Boston Properties v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction\n",
    "Data is drawn from the [Boston Property Assessment Dataset](https://data.boston.gov/dataset/property-assessment), which also provides a data key.\n",
    "\n",
    "Because of the large size of the dataset, only residential properties were used when fitting models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# libraries and settings\n",
    "import datetime as dt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns; sns.set()\n",
    "import matplotlib.pyplot as plt\n",
    "import uszipcode # zip code data, will download on first import after install via pip\n",
    "from uszipcode import SearchEngine\n",
    "from IPython.display import display, clear_output\n",
    "# suppress future warnings\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "# display settings\n",
    "pd.options.display.max_columns = None\n",
    "pd.options.mode.chained_assignment = None  # default='warn'\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dtype</th>\n",
       "      <th>is_luxury</th>\n",
       "      <th>always_drop</th>\n",
       "      <th>commercial_only</th>\n",
       "      <th>value_dict</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>col</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PID</th>\n",
       "      <td>int64</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CM_ID</th>\n",
       "      <td>float64</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>only useful when looking at units inside a giv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GIS_ID</th>\n",
       "      <td>int64</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ST_NUM</th>\n",
       "      <td>str</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ST_NAME</th>\n",
       "      <td>str</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           dtype  is_luxury  always_drop  commercial_only value_dict  \\\n",
       "col                                                                    \n",
       "PID        int64      False        False            False        NaN   \n",
       "CM_ID    float64      False        False            False        NaN   \n",
       "GIS_ID     int64      False         True            False        NaN   \n",
       "ST_NUM       str      False         True            False        NaN   \n",
       "ST_NAME      str      False         True            False        NaN   \n",
       "\n",
       "                                                   comment  \n",
       "col                                                         \n",
       "PID                                                    NaN  \n",
       "CM_ID    only useful when looking at units inside a giv...  \n",
       "GIS_ID                                                 NaN  \n",
       "ST_NUM                                                 NaN  \n",
       "ST_NAME                                                NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# read dtype and specification table from Excel\n",
    "propDtypes = pd.read_excel('prop_dtypes.xlsx', index_col='col')\n",
    "# fill na values in boolean cols\n",
    "bool_cols = ['is_luxury', 'always_drop', 'commercial_only']\n",
    "propDtypes[bool_cols] = propDtypes[bool_cols].fillna(False).astype('bool')\n",
    "display(propDtypes.head(5))\n",
    "# convert into dictionary for use with read_csv\n",
    "dtDict = propDtypes['dtype'].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read raw data, allow implicit type detection even though you provide types\n",
    "prop = pd.read_csv('fy19fullpropassess.csv', low_memory=False,\n",
    "                  dtype=dtDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load Boston zip codes to map neighborhoods\n",
    "zipNbh = pd.read_excel('zipNeighborhoods.xlsx',\n",
    "                       converters={'zip': str, 'Neighborhood': str})\n",
    "zipNbh.set_index('zip', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example Property Data:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PID</th>\n",
       "      <th>CM_ID</th>\n",
       "      <th>GIS_ID</th>\n",
       "      <th>ST_NUM</th>\n",
       "      <th>ST_NAME</th>\n",
       "      <th>ST_NAME_SUF</th>\n",
       "      <th>UNIT_NUM</th>\n",
       "      <th>ZIPCODE</th>\n",
       "      <th>PTYPE</th>\n",
       "      <th>LU</th>\n",
       "      <th>OWN_OCC</th>\n",
       "      <th>OWNER</th>\n",
       "      <th>MAIL_ADDRESSEE</th>\n",
       "      <th>MAIL_ADDRESS</th>\n",
       "      <th>MAIL CS</th>\n",
       "      <th>MAIL_ZIPCODE</th>\n",
       "      <th>AV_LAND</th>\n",
       "      <th>AV_BLDG</th>\n",
       "      <th>AV_TOTAL</th>\n",
       "      <th>GROSS_TAX</th>\n",
       "      <th>LAND_SF</th>\n",
       "      <th>YR_BUILT</th>\n",
       "      <th>YR_REMOD</th>\n",
       "      <th>GROSS_AREA</th>\n",
       "      <th>LIVING_AREA</th>\n",
       "      <th>NUM_FLOORS</th>\n",
       "      <th>STRUCTURE_CLASS</th>\n",
       "      <th>R_BLDG_STYL</th>\n",
       "      <th>R_ROOF_TYP</th>\n",
       "      <th>R_EXT_FIN</th>\n",
       "      <th>R_TOTAL_RMS</th>\n",
       "      <th>R_BDRMS</th>\n",
       "      <th>R_FULL_BTH</th>\n",
       "      <th>R_HALF_BTH</th>\n",
       "      <th>R_BTH_STYLE</th>\n",
       "      <th>R_BTH_STYLE2</th>\n",
       "      <th>R_BTH_STYLE3</th>\n",
       "      <th>R_KITCH</th>\n",
       "      <th>R_KITCH_STYLE</th>\n",
       "      <th>R_KITCH_STYLE2</th>\n",
       "      <th>R_KITCH_STYLE3</th>\n",
       "      <th>R_HEAT_TYP</th>\n",
       "      <th>R_AC</th>\n",
       "      <th>R_FPLACE</th>\n",
       "      <th>R_EXT_CND</th>\n",
       "      <th>R_OVRALL_CND</th>\n",
       "      <th>R_INT_CND</th>\n",
       "      <th>R_INT_FIN</th>\n",
       "      <th>R_VIEW</th>\n",
       "      <th>S_NUM_BLDG</th>\n",
       "      <th>S_BLDG_STYL</th>\n",
       "      <th>S_UNIT_RES</th>\n",
       "      <th>S_UNIT_COM</th>\n",
       "      <th>S_UNIT_RC</th>\n",
       "      <th>S_EXT_FIN</th>\n",
       "      <th>S_EXT_CND</th>\n",
       "      <th>U_BASE_FLOOR</th>\n",
       "      <th>U_NUM_PARK</th>\n",
       "      <th>U_CORNER</th>\n",
       "      <th>U_ORIENT</th>\n",
       "      <th>U_TOT_RMS</th>\n",
       "      <th>U_BDRMS</th>\n",
       "      <th>U_FULL_BTH</th>\n",
       "      <th>U_HALF_BTH</th>\n",
       "      <th>U_BTH_STYLE</th>\n",
       "      <th>U_BTH_STYLE2</th>\n",
       "      <th>U_BTH_STYLE3</th>\n",
       "      <th>U_KITCH_TYPE</th>\n",
       "      <th>U_KITCH_STYLE</th>\n",
       "      <th>U_HEAT_TYP</th>\n",
       "      <th>U_AC</th>\n",
       "      <th>U_FPLACE</th>\n",
       "      <th>U_INT_FIN</th>\n",
       "      <th>U_INT_CND</th>\n",
       "      <th>U_VIEW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100209</th>\n",
       "      <td>1404559004</td>\n",
       "      <td>1.404559e+09</td>\n",
       "      <td>1404559000</td>\n",
       "      <td>31</td>\n",
       "      <td>HAVELOCK</td>\n",
       "      <td>ST</td>\n",
       "      <td>2</td>\n",
       "      <td>2124</td>\n",
       "      <td>102</td>\n",
       "      <td>CD</td>\n",
       "      <td>N</td>\n",
       "      <td>BARRETT NJERI N</td>\n",
       "      <td>C/O NJERI BARRETT</td>\n",
       "      <td>31 HAVELOCK ST #2</td>\n",
       "      <td>DORCHESTER MA</td>\n",
       "      <td>2124</td>\n",
       "      <td>0</td>\n",
       "      <td>182400</td>\n",
       "      <td>182400</td>\n",
       "      <td>192250</td>\n",
       "      <td>987.0</td>\n",
       "      <td>1905.0</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>987.0</td>\n",
       "      <td>987.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>T</td>\n",
       "      <td>6</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>S</td>\n",
       "      <td>W</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66852</th>\n",
       "      <td>701333000</td>\n",
       "      <td>7.013330e+08</td>\n",
       "      <td>701333000</td>\n",
       "      <td>62</td>\n",
       "      <td>GATES</td>\n",
       "      <td>ST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2127</td>\n",
       "      <td>995</td>\n",
       "      <td>CM</td>\n",
       "      <td>N</td>\n",
       "      <td>SIXTY2 GATES ST CONDO TR</td>\n",
       "      <td></td>\n",
       "      <td>62 GATES ST</td>\n",
       "      <td>S BOSTON MA</td>\n",
       "      <td>2127</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1920.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>R</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LR</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>W</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107462</th>\n",
       "      <td>1602046000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1602046000</td>\n",
       "      <td>24 26</td>\n",
       "      <td>GARNER</td>\n",
       "      <td>RD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2122</td>\n",
       "      <td>104</td>\n",
       "      <td>R2</td>\n",
       "      <td>Y</td>\n",
       "      <td>ZIMMER JAMES M</td>\n",
       "      <td></td>\n",
       "      <td>24 GARNER RD</td>\n",
       "      <td>DORCHESTER MA</td>\n",
       "      <td>2122</td>\n",
       "      <td>133600</td>\n",
       "      <td>431800</td>\n",
       "      <td>565400</td>\n",
       "      <td>595932</td>\n",
       "      <td>3404.0</td>\n",
       "      <td>1920.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4443.0</td>\n",
       "      <td>2682.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>R</td>\n",
       "      <td>TF</td>\n",
       "      <td>H</td>\n",
       "      <td>M</td>\n",
       "      <td>12.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>S</td>\n",
       "      <td>S</td>\n",
       "      <td>S</td>\n",
       "      <td>2.0</td>\n",
       "      <td>S</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>W</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22233</th>\n",
       "      <td>303029332</td>\n",
       "      <td>3.030292e+08</td>\n",
       "      <td>303029200</td>\n",
       "      <td>28 32</td>\n",
       "      <td>ATLANTIC</td>\n",
       "      <td>AV</td>\n",
       "      <td>327</td>\n",
       "      <td>2110</td>\n",
       "      <td>102</td>\n",
       "      <td>CD</td>\n",
       "      <td>Y</td>\n",
       "      <td>LAUER-WILLIAMS JEANNE TS</td>\n",
       "      <td></td>\n",
       "      <td>28 ATLANTIC AV #327</td>\n",
       "      <td>BOSTON MA</td>\n",
       "      <td>2110</td>\n",
       "      <td>0</td>\n",
       "      <td>1032600</td>\n",
       "      <td>1032600</td>\n",
       "      <td>1088361</td>\n",
       "      <td>1235.0</td>\n",
       "      <td>1850.0</td>\n",
       "      <td>1973.0</td>\n",
       "      <td>1235.0</td>\n",
       "      <td>1235.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>M</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>M</td>\n",
       "      <td>F</td>\n",
       "      <td>C</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10550</th>\n",
       "      <td>200905006</td>\n",
       "      <td>2.009050e+08</td>\n",
       "      <td>200905000</td>\n",
       "      <td>52</td>\n",
       "      <td>SULLIVAN</td>\n",
       "      <td>ST</td>\n",
       "      <td>3</td>\n",
       "      <td>2129</td>\n",
       "      <td>102</td>\n",
       "      <td>CD</td>\n",
       "      <td>N</td>\n",
       "      <td>MURRMAN JOANNE</td>\n",
       "      <td></td>\n",
       "      <td>52 SULLIVAN ST #3</td>\n",
       "      <td>CHARLESTOWN MA</td>\n",
       "      <td>2129</td>\n",
       "      <td>0</td>\n",
       "      <td>407900</td>\n",
       "      <td>407900</td>\n",
       "      <td>429927</td>\n",
       "      <td>1235.0</td>\n",
       "      <td>1857.0</td>\n",
       "      <td>1985.0</td>\n",
       "      <td>1235.0</td>\n",
       "      <td>680.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>T</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>M</td>\n",
       "      <td>W</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               PID         CM_ID      GIS_ID ST_NUM   ST_NAME ST_NAME_SUF  \\\n",
       "100209  1404559004  1.404559e+09  1404559000     31  HAVELOCK          ST   \n",
       "66852    701333000  7.013330e+08   701333000     62     GATES          ST   \n",
       "107462  1602046000           NaN  1602046000  24 26    GARNER          RD   \n",
       "22233    303029332  3.030292e+08   303029200  28 32  ATLANTIC          AV   \n",
       "10550    200905006  2.009050e+08   200905000     52  SULLIVAN          ST   \n",
       "\n",
       "       UNIT_NUM ZIPCODE  PTYPE  LU OWN_OCC                     OWNER  \\\n",
       "100209        2    2124    102  CD       N           BARRETT NJERI N   \n",
       "66852       NaN    2127    995  CM       N  SIXTY2 GATES ST CONDO TR   \n",
       "107462      NaN    2122    104  R2       Y            ZIMMER JAMES M   \n",
       "22233       327    2110    102  CD       Y  LAUER-WILLIAMS JEANNE TS   \n",
       "10550         3    2129    102  CD       N            MURRMAN JOANNE   \n",
       "\n",
       "           MAIL_ADDRESSEE         MAIL_ADDRESS         MAIL CS MAIL_ZIPCODE  \\\n",
       "100209  C/O NJERI BARRETT    31 HAVELOCK ST #2   DORCHESTER MA         2124   \n",
       "66852                              62 GATES ST     S BOSTON MA         2127   \n",
       "107462                            24 GARNER RD   DORCHESTER MA         2122   \n",
       "22233                      28 ATLANTIC AV #327       BOSTON MA         2110   \n",
       "10550                        52 SULLIVAN ST #3  CHARLESTOWN MA         2129   \n",
       "\n",
       "        AV_LAND  AV_BLDG  AV_TOTAL  GROSS_TAX  LAND_SF  YR_BUILT  YR_REMOD  \\\n",
       "100209        0   182400    182400     192250    987.0    1905.0    1987.0   \n",
       "66852         0        0         0          0   1920.0    1900.0    2000.0   \n",
       "107462   133600   431800    565400     595932   3404.0    1920.0       0.0   \n",
       "22233         0  1032600   1032600    1088361   1235.0    1850.0    1973.0   \n",
       "10550         0   407900    407900     429927   1235.0    1857.0    1985.0   \n",
       "\n",
       "        GROSS_AREA  LIVING_AREA  NUM_FLOORS STRUCTURE_CLASS R_BLDG_STYL  \\\n",
       "100209       987.0        987.0         1.0             NaN         NaN   \n",
       "66852          0.0          0.0         3.0               R         NaN   \n",
       "107462      4443.0       2682.0         2.0               R          TF   \n",
       "22233       1235.0       1235.0         1.0             NaN         NaN   \n",
       "10550       1235.0        680.0         1.0             NaN         NaN   \n",
       "\n",
       "       R_ROOF_TYP R_EXT_FIN  R_TOTAL_RMS  R_BDRMS  R_FULL_BTH  R_HALF_BTH  \\\n",
       "100209        NaN       NaN          NaN      NaN         NaN         NaN   \n",
       "66852         NaN       NaN          NaN      NaN         NaN         NaN   \n",
       "107462          H         M         12.0      6.0         2.0         2.0   \n",
       "22233         NaN       NaN          NaN      NaN         NaN         NaN   \n",
       "10550         NaN       NaN          NaN      NaN         NaN         NaN   \n",
       "\n",
       "       R_BTH_STYLE R_BTH_STYLE2 R_BTH_STYLE3  R_KITCH R_KITCH_STYLE  \\\n",
       "100209         NaN          NaN          NaN      NaN           NaN   \n",
       "66852          NaN          NaN          NaN      NaN           NaN   \n",
       "107462           S            S            S      2.0             S   \n",
       "22233          NaN          NaN          NaN      NaN           NaN   \n",
       "10550          NaN          NaN          NaN      NaN           NaN   \n",
       "\n",
       "       R_KITCH_STYLE2 R_KITCH_STYLE3 R_HEAT_TYP R_AC  R_FPLACE R_EXT_CND  \\\n",
       "100209            NaN            NaN        NaN  NaN       NaN       NaN   \n",
       "66852             NaN            NaN        NaN  NaN       NaN       NaN   \n",
       "107462              S            NaN          W    N       0.0         A   \n",
       "22233             NaN            NaN        NaN  NaN       NaN       NaN   \n",
       "10550             NaN            NaN        NaN  NaN       NaN       NaN   \n",
       "\n",
       "       R_OVRALL_CND R_INT_CND R_INT_FIN R_VIEW  S_NUM_BLDG S_BLDG_STYL  \\\n",
       "100209          NaN       NaN       NaN    NaN         NaN         NaN   \n",
       "66852           NaN       NaN       NaN    NaN         NaN          LR   \n",
       "107462            A         A         N      A         NaN         NaN   \n",
       "22233           NaN       NaN       NaN    NaN         NaN         NaN   \n",
       "10550           NaN       NaN       NaN    NaN         NaN         NaN   \n",
       "\n",
       "        S_UNIT_RES  S_UNIT_COM  S_UNIT_RC S_EXT_FIN S_EXT_CND  U_BASE_FLOOR  \\\n",
       "100209         NaN         NaN        NaN       NaN       NaN           2.0   \n",
       "66852          3.0         0.0        0.0         W         A           NaN   \n",
       "107462         NaN         NaN        NaN       NaN       NaN           NaN   \n",
       "22233          NaN         NaN        NaN       NaN       NaN           3.0   \n",
       "10550          NaN         NaN        NaN       NaN       NaN           3.0   \n",
       "\n",
       "        U_NUM_PARK U_CORNER U_ORIENT U_TOT_RMS  U_BDRMS  U_FULL_BTH  \\\n",
       "100209         0.0        N        T         6      3.0         1.0   \n",
       "66852          NaN      NaN      NaN       NaN      NaN         NaN   \n",
       "107462         NaN      NaN      NaN       NaN      NaN         NaN   \n",
       "22233          0.0        N        A         4      2.0         2.0   \n",
       "10550          0.0        N        T         3      1.0         1.0   \n",
       "\n",
       "        U_HALF_BTH U_BTH_STYLE U_BTH_STYLE2 U_BTH_STYLE3 U_KITCH_TYPE  \\\n",
       "100209         0.0           S          NaN          NaN            F   \n",
       "66852          NaN         NaN          NaN          NaN          NaN   \n",
       "107462         NaN         NaN          NaN          NaN          NaN   \n",
       "22233          0.0           M            M          NaN            O   \n",
       "10550          0.0           M          NaN          NaN            F   \n",
       "\n",
       "       U_KITCH_STYLE U_HEAT_TYP U_AC  U_FPLACE U_INT_FIN U_INT_CND U_VIEW  \n",
       "100209             S          W    N       0.0         N         A      A  \n",
       "66852            NaN        NaN  NaN       NaN       NaN       NaN    NaN  \n",
       "107462           NaN        NaN  NaN       NaN       NaN       NaN    NaN  \n",
       "22233              M          F    C       0.0         N         A      G  \n",
       "10550              M          W    N       0.0         N         A      A  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example Zipcode Data:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Neighborhood</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zip</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>02128</th>\n",
       "      <td>East Boston</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02447</th>\n",
       "      <td>Brookline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02131</th>\n",
       "      <td>Roslindale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02111</th>\n",
       "      <td>Boston Chinatown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02445</th>\n",
       "      <td>Brookline</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Neighborhood\n",
       "zip                    \n",
       "02128       East Boston\n",
       "02447         Brookline\n",
       "02131        Roslindale\n",
       "02111  Boston Chinatown\n",
       "02445         Brookline"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# show samples of property and zipcode data\n",
    "print('Example Property Data:')\n",
    "display(prop.sample(5))\n",
    "print('Example Zipcode Data:')\n",
    "display(zipNbh.sample(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleanup and Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "columns to be dropped:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['GIS_ID',\n",
       " 'ST_NUM',\n",
       " 'ST_NAME',\n",
       " 'ST_NAME_SUF',\n",
       " 'UNIT_NUM',\n",
       " 'PTYPE',\n",
       " 'MAIL_ADDRESSEE',\n",
       " 'MAIL_ADDRESS',\n",
       " 'MAIL CS',\n",
       " 'GROSS_AREA',\n",
       " 'STRUCTURE_CLASS']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PID</th>\n",
       "      <th>CM_ID</th>\n",
       "      <th>ZIPCODE</th>\n",
       "      <th>LU</th>\n",
       "      <th>OWN_OCC</th>\n",
       "      <th>OWNER</th>\n",
       "      <th>MAIL_ZIPCODE</th>\n",
       "      <th>AV_LAND</th>\n",
       "      <th>AV_BLDG</th>\n",
       "      <th>AV_TOTAL</th>\n",
       "      <th>GROSS_TAX</th>\n",
       "      <th>LAND_SF</th>\n",
       "      <th>YR_BUILT</th>\n",
       "      <th>YR_REMOD</th>\n",
       "      <th>LIVING_AREA</th>\n",
       "      <th>NUM_FLOORS</th>\n",
       "      <th>R_BLDG_STYL</th>\n",
       "      <th>R_ROOF_TYP</th>\n",
       "      <th>R_EXT_FIN</th>\n",
       "      <th>R_TOTAL_RMS</th>\n",
       "      <th>R_BDRMS</th>\n",
       "      <th>R_FULL_BTH</th>\n",
       "      <th>R_HALF_BTH</th>\n",
       "      <th>R_BTH_STYLE</th>\n",
       "      <th>R_BTH_STYLE2</th>\n",
       "      <th>R_BTH_STYLE3</th>\n",
       "      <th>R_KITCH</th>\n",
       "      <th>R_KITCH_STYLE</th>\n",
       "      <th>R_KITCH_STYLE2</th>\n",
       "      <th>R_KITCH_STYLE3</th>\n",
       "      <th>R_HEAT_TYP</th>\n",
       "      <th>R_AC</th>\n",
       "      <th>R_FPLACE</th>\n",
       "      <th>R_EXT_CND</th>\n",
       "      <th>R_OVRALL_CND</th>\n",
       "      <th>R_INT_CND</th>\n",
       "      <th>R_INT_FIN</th>\n",
       "      <th>R_VIEW</th>\n",
       "      <th>S_NUM_BLDG</th>\n",
       "      <th>S_BLDG_STYL</th>\n",
       "      <th>S_UNIT_RES</th>\n",
       "      <th>S_UNIT_COM</th>\n",
       "      <th>S_UNIT_RC</th>\n",
       "      <th>S_EXT_FIN</th>\n",
       "      <th>S_EXT_CND</th>\n",
       "      <th>U_BASE_FLOOR</th>\n",
       "      <th>U_NUM_PARK</th>\n",
       "      <th>U_CORNER</th>\n",
       "      <th>U_ORIENT</th>\n",
       "      <th>U_TOT_RMS</th>\n",
       "      <th>U_BDRMS</th>\n",
       "      <th>U_FULL_BTH</th>\n",
       "      <th>U_HALF_BTH</th>\n",
       "      <th>U_BTH_STYLE</th>\n",
       "      <th>U_BTH_STYLE2</th>\n",
       "      <th>U_BTH_STYLE3</th>\n",
       "      <th>U_KITCH_TYPE</th>\n",
       "      <th>U_KITCH_STYLE</th>\n",
       "      <th>U_HEAT_TYP</th>\n",
       "      <th>U_AC</th>\n",
       "      <th>U_FPLACE</th>\n",
       "      <th>U_INT_FIN</th>\n",
       "      <th>U_INT_CND</th>\n",
       "      <th>U_VIEW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>151372</th>\n",
       "      <td>2007013116</td>\n",
       "      <td>2.007013e+09</td>\n",
       "      <td>2467</td>\n",
       "      <td>CD</td>\n",
       "      <td>Y</td>\n",
       "      <td>LIEBMAN DEBORAH</td>\n",
       "      <td>2467</td>\n",
       "      <td>0</td>\n",
       "      <td>279600</td>\n",
       "      <td>279600</td>\n",
       "      <td>294699</td>\n",
       "      <td>793.0</td>\n",
       "      <td>1974.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>793.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Y</td>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>M</td>\n",
       "      <td>F</td>\n",
       "      <td>C</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52769</th>\n",
       "      <td>504049000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2215</td>\n",
       "      <td>E</td>\n",
       "      <td>N</td>\n",
       "      <td>BOSTON UNIVERSITY TRS OF</td>\n",
       "      <td>2215</td>\n",
       "      <td>767200</td>\n",
       "      <td>1454300</td>\n",
       "      <td>2221500</td>\n",
       "      <td>0</td>\n",
       "      <td>3156.0</td>\n",
       "      <td>1899.0</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>7032.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95414</th>\n",
       "      <td>1400064000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2121</td>\n",
       "      <td>A</td>\n",
       "      <td>N</td>\n",
       "      <td>SECRETARY OF HOUSING</td>\n",
       "      <td>2109</td>\n",
       "      <td>206127</td>\n",
       "      <td>345217</td>\n",
       "      <td>551344</td>\n",
       "      <td>581117</td>\n",
       "      <td>6812.0</td>\n",
       "      <td>1925.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9246.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136573</th>\n",
       "      <td>1901474000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2130</td>\n",
       "      <td>R1</td>\n",
       "      <td>Y</td>\n",
       "      <td>GRACEFFA LIA M</td>\n",
       "      <td>2130</td>\n",
       "      <td>125200</td>\n",
       "      <td>408100</td>\n",
       "      <td>533300</td>\n",
       "      <td>562098</td>\n",
       "      <td>1842.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>1950.0</td>\n",
       "      <td>1755.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>CL</td>\n",
       "      <td>G</td>\n",
       "      <td>F</td>\n",
       "      <td>8.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>S</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>W</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90816</th>\n",
       "      <td>1300091000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2125</td>\n",
       "      <td>E</td>\n",
       "      <td>N</td>\n",
       "      <td>DNI 4 INC</td>\n",
       "      <td>2119</td>\n",
       "      <td>25400</td>\n",
       "      <td>0</td>\n",
       "      <td>25400</td>\n",
       "      <td>0</td>\n",
       "      <td>2776.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               PID         CM_ID ZIPCODE  LU OWN_OCC  \\\n",
       "151372  2007013116  2.007013e+09    2467  CD       Y   \n",
       "52769    504049000           NaN    2215   E       N   \n",
       "95414   1400064000           NaN    2121   A       N   \n",
       "136573  1901474000           NaN    2130  R1       Y   \n",
       "90816   1300091000           NaN    2125   E       N   \n",
       "\n",
       "                           OWNER MAIL_ZIPCODE  AV_LAND  AV_BLDG  AV_TOTAL  \\\n",
       "151372           LIEBMAN DEBORAH         2467        0   279600    279600   \n",
       "52769   BOSTON UNIVERSITY TRS OF         2215   767200  1454300   2221500   \n",
       "95414       SECRETARY OF HOUSING         2109   206127   345217    551344   \n",
       "136573            GRACEFFA LIA M         2130   125200   408100    533300   \n",
       "90816                  DNI 4 INC         2119    25400        0     25400   \n",
       "\n",
       "        GROSS_TAX  LAND_SF  YR_BUILT  YR_REMOD  LIVING_AREA  NUM_FLOORS  \\\n",
       "151372     294699    793.0    1974.0       0.0        793.0         1.0   \n",
       "52769           0   3156.0    1899.0    2002.0       7032.0         4.0   \n",
       "95414      581117   6812.0    1925.0       NaN       9246.0         3.0   \n",
       "136573     562098   1842.0    1900.0    1950.0       1755.0         2.0   \n",
       "90816           0   2776.0       0.0       NaN          0.0         NaN   \n",
       "\n",
       "       R_BLDG_STYL R_ROOF_TYP R_EXT_FIN  R_TOTAL_RMS  R_BDRMS  R_FULL_BTH  \\\n",
       "151372         NaN        NaN       NaN          NaN      NaN         NaN   \n",
       "52769          NaN        NaN       NaN          NaN      NaN         NaN   \n",
       "95414          NaN        NaN       NaN          NaN      NaN         NaN   \n",
       "136573          CL          G         F          8.0      3.0         1.0   \n",
       "90816          NaN        NaN       NaN          NaN      NaN         NaN   \n",
       "\n",
       "        R_HALF_BTH R_BTH_STYLE R_BTH_STYLE2 R_BTH_STYLE3  R_KITCH  \\\n",
       "151372         NaN         NaN          NaN          NaN      NaN   \n",
       "52769          NaN         NaN          NaN          NaN      NaN   \n",
       "95414          NaN         NaN          NaN          NaN      NaN   \n",
       "136573         1.0           S            S          NaN      1.0   \n",
       "90816          NaN         NaN          NaN          NaN      NaN   \n",
       "\n",
       "       R_KITCH_STYLE R_KITCH_STYLE2 R_KITCH_STYLE3 R_HEAT_TYP R_AC  R_FPLACE  \\\n",
       "151372           NaN            NaN            NaN        NaN  NaN       NaN   \n",
       "52769            NaN            NaN            NaN        NaN  NaN       NaN   \n",
       "95414            NaN            NaN            NaN        NaN  NaN       NaN   \n",
       "136573             S            NaN            NaN          W    N       0.0   \n",
       "90816            NaN            NaN            NaN        NaN  NaN       NaN   \n",
       "\n",
       "       R_EXT_CND R_OVRALL_CND R_INT_CND R_INT_FIN R_VIEW  S_NUM_BLDG  \\\n",
       "151372       NaN          NaN       NaN       NaN    NaN         NaN   \n",
       "52769        NaN          NaN       NaN       NaN    NaN         NaN   \n",
       "95414        NaN          NaN       NaN       NaN    NaN         NaN   \n",
       "136573         A            A         A         N      A         NaN   \n",
       "90816        NaN          NaN       NaN       NaN    NaN         NaN   \n",
       "\n",
       "       S_BLDG_STYL  S_UNIT_RES  S_UNIT_COM  S_UNIT_RC S_EXT_FIN S_EXT_CND  \\\n",
       "151372         NaN         NaN         NaN        NaN       NaN       NaN   \n",
       "52769          NaN         NaN         NaN        NaN       NaN       NaN   \n",
       "95414          NaN         NaN         NaN        NaN       NaN       NaN   \n",
       "136573         NaN         NaN         NaN        NaN       NaN       NaN   \n",
       "90816          NaN         NaN         NaN        NaN       NaN       NaN   \n",
       "\n",
       "        U_BASE_FLOOR  U_NUM_PARK U_CORNER U_ORIENT U_TOT_RMS  U_BDRMS  \\\n",
       "151372           2.0         1.0        Y        A         3      1.0   \n",
       "52769            NaN         NaN      NaN      NaN       NaN      NaN   \n",
       "95414            NaN         NaN      NaN      NaN       NaN      NaN   \n",
       "136573           NaN         NaN      NaN      NaN       NaN      NaN   \n",
       "90816            NaN         NaN      NaN      NaN       NaN      NaN   \n",
       "\n",
       "        U_FULL_BTH  U_HALF_BTH U_BTH_STYLE U_BTH_STYLE2 U_BTH_STYLE3  \\\n",
       "151372         1.0         0.0           M          NaN          NaN   \n",
       "52769          NaN         NaN         NaN          NaN          NaN   \n",
       "95414          NaN         NaN         NaN          NaN          NaN   \n",
       "136573         NaN         NaN         NaN          NaN          NaN   \n",
       "90816          NaN         NaN         NaN          NaN          NaN   \n",
       "\n",
       "       U_KITCH_TYPE U_KITCH_STYLE U_HEAT_TYP U_AC  U_FPLACE U_INT_FIN  \\\n",
       "151372            O             M          F    C       0.0         N   \n",
       "52769           NaN           NaN        NaN  NaN       NaN       NaN   \n",
       "95414           NaN           NaN        NaN  NaN       NaN       NaN   \n",
       "136573          NaN           NaN        NaN  NaN       NaN       NaN   \n",
       "90816           NaN           NaN        NaN  NaN       NaN       NaN   \n",
       "\n",
       "       U_INT_CND U_VIEW  \n",
       "151372         A      A  \n",
       "52769        NaN    NaN  \n",
       "95414        NaN    NaN  \n",
       "136573       NaN    NaN  \n",
       "90816        NaN    NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# drop columns that aren't going to be used in this analysis\n",
    "dropCols = list(propDtypes.loc[propDtypes['always_drop'] | propDtypes['commercial_only']].index)\n",
    "print('columns to be dropped:')\n",
    "display(dropCols)\n",
    "prop = prop.drop(columns=list(dropCols), axis=1)\n",
    "display(prop.sample(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zip codes after prepending leading zero:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    02108\n",
       "1    02108\n",
       "2    02108\n",
       "3    02108\n",
       "4    02108\n",
       "5    02108\n",
       "6    02108\n",
       "7    02108\n",
       "8    02108\n",
       "9    02108\n",
       "Name: ZIPCODE, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PID</th>\n",
       "      <th>CM_ID</th>\n",
       "      <th>ZIPCODE</th>\n",
       "      <th>LU</th>\n",
       "      <th>OWN_OCC</th>\n",
       "      <th>OWNER</th>\n",
       "      <th>MAIL_ZIPCODE</th>\n",
       "      <th>AV_LAND</th>\n",
       "      <th>AV_BLDG</th>\n",
       "      <th>AV_TOTAL</th>\n",
       "      <th>GROSS_TAX</th>\n",
       "      <th>LAND_SF</th>\n",
       "      <th>YR_BUILT</th>\n",
       "      <th>YR_REMOD</th>\n",
       "      <th>LIVING_AREA</th>\n",
       "      <th>NUM_FLOORS</th>\n",
       "      <th>R_BLDG_STYL</th>\n",
       "      <th>R_ROOF_TYP</th>\n",
       "      <th>R_EXT_FIN</th>\n",
       "      <th>R_TOTAL_RMS</th>\n",
       "      <th>R_BDRMS</th>\n",
       "      <th>R_FULL_BTH</th>\n",
       "      <th>R_HALF_BTH</th>\n",
       "      <th>R_BTH_STYLE</th>\n",
       "      <th>R_BTH_STYLE2</th>\n",
       "      <th>R_BTH_STYLE3</th>\n",
       "      <th>R_KITCH</th>\n",
       "      <th>R_KITCH_STYLE</th>\n",
       "      <th>R_KITCH_STYLE2</th>\n",
       "      <th>R_KITCH_STYLE3</th>\n",
       "      <th>R_HEAT_TYP</th>\n",
       "      <th>R_AC</th>\n",
       "      <th>R_FPLACE</th>\n",
       "      <th>R_EXT_CND</th>\n",
       "      <th>R_OVRALL_CND</th>\n",
       "      <th>R_INT_CND</th>\n",
       "      <th>R_INT_FIN</th>\n",
       "      <th>R_VIEW</th>\n",
       "      <th>S_NUM_BLDG</th>\n",
       "      <th>S_BLDG_STYL</th>\n",
       "      <th>S_UNIT_RES</th>\n",
       "      <th>S_UNIT_COM</th>\n",
       "      <th>S_UNIT_RC</th>\n",
       "      <th>S_EXT_FIN</th>\n",
       "      <th>S_EXT_CND</th>\n",
       "      <th>U_BASE_FLOOR</th>\n",
       "      <th>U_NUM_PARK</th>\n",
       "      <th>U_CORNER</th>\n",
       "      <th>U_ORIENT</th>\n",
       "      <th>U_TOT_RMS</th>\n",
       "      <th>U_BDRMS</th>\n",
       "      <th>U_FULL_BTH</th>\n",
       "      <th>U_HALF_BTH</th>\n",
       "      <th>U_BTH_STYLE</th>\n",
       "      <th>U_BTH_STYLE2</th>\n",
       "      <th>U_BTH_STYLE3</th>\n",
       "      <th>U_KITCH_TYPE</th>\n",
       "      <th>U_KITCH_STYLE</th>\n",
       "      <th>U_HEAT_TYP</th>\n",
       "      <th>U_AC</th>\n",
       "      <th>U_FPLACE</th>\n",
       "      <th>U_INT_FIN</th>\n",
       "      <th>U_INT_CND</th>\n",
       "      <th>U_VIEW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>131764</th>\n",
       "      <td>1811774008</td>\n",
       "      <td>1.811774e+09</td>\n",
       "      <td>02136</td>\n",
       "      <td>CP</td>\n",
       "      <td>N</td>\n",
       "      <td>TEAL JUSTIN</td>\n",
       "      <td>2136</td>\n",
       "      <td>0</td>\n",
       "      <td>6400</td>\n",
       "      <td>6400</td>\n",
       "      <td>6746</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129284</th>\n",
       "      <td>1809648001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02136</td>\n",
       "      <td>E</td>\n",
       "      <td>N</td>\n",
       "      <td>COMMONWEALTH OF MASS</td>\n",
       "      <td>2136</td>\n",
       "      <td>6369800</td>\n",
       "      <td>0</td>\n",
       "      <td>6369800</td>\n",
       "      <td>0</td>\n",
       "      <td>840339.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160554</th>\n",
       "      <td>2101318000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02134</td>\n",
       "      <td>R1</td>\n",
       "      <td>N</td>\n",
       "      <td>REBAA LLC</td>\n",
       "      <td>2134</td>\n",
       "      <td>165600</td>\n",
       "      <td>375700</td>\n",
       "      <td>541300</td>\n",
       "      <td>570530</td>\n",
       "      <td>3363.0</td>\n",
       "      <td>1895.0</td>\n",
       "      <td>1999.0</td>\n",
       "      <td>2106.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>CL</td>\n",
       "      <td>H</td>\n",
       "      <td>M</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>S</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               PID         CM_ID ZIPCODE  LU OWN_OCC                 OWNER  \\\n",
       "131764  1811774008  1.811774e+09   02136  CP       N           TEAL JUSTIN   \n",
       "129284  1809648001           NaN   02136   E       N  COMMONWEALTH OF MASS   \n",
       "160554  2101318000           NaN   02134  R1       N             REBAA LLC   \n",
       "\n",
       "       MAIL_ZIPCODE  AV_LAND  AV_BLDG  AV_TOTAL  GROSS_TAX   LAND_SF  \\\n",
       "131764         2136        0     6400      6400       6746       NaN   \n",
       "129284         2136  6369800        0   6369800          0  840339.0   \n",
       "160554         2134   165600   375700    541300     570530    3363.0   \n",
       "\n",
       "        YR_BUILT  YR_REMOD  LIVING_AREA  NUM_FLOORS R_BLDG_STYL R_ROOF_TYP  \\\n",
       "131764       NaN       NaN          NaN         NaN         NaN        NaN   \n",
       "129284       0.0       NaN          0.0         NaN         NaN        NaN   \n",
       "160554    1895.0    1999.0       2106.0         2.5          CL          H   \n",
       "\n",
       "       R_EXT_FIN  R_TOTAL_RMS  R_BDRMS  R_FULL_BTH  R_HALF_BTH R_BTH_STYLE  \\\n",
       "131764       NaN          NaN      NaN         NaN         NaN         NaN   \n",
       "129284       NaN          NaN      NaN         NaN         NaN         NaN   \n",
       "160554         M          6.0      3.0         2.0         0.0           S   \n",
       "\n",
       "       R_BTH_STYLE2 R_BTH_STYLE3  R_KITCH R_KITCH_STYLE R_KITCH_STYLE2  \\\n",
       "131764          NaN          NaN      NaN           NaN            NaN   \n",
       "129284          NaN          NaN      NaN           NaN            NaN   \n",
       "160554            S          NaN      1.0             S            NaN   \n",
       "\n",
       "       R_KITCH_STYLE3 R_HEAT_TYP R_AC  R_FPLACE R_EXT_CND R_OVRALL_CND  \\\n",
       "131764            NaN        NaN  NaN       NaN       NaN          NaN   \n",
       "129284            NaN        NaN  NaN       NaN       NaN          NaN   \n",
       "160554            NaN          F    N       0.0         A            A   \n",
       "\n",
       "       R_INT_CND R_INT_FIN R_VIEW  S_NUM_BLDG S_BLDG_STYL  S_UNIT_RES  \\\n",
       "131764       NaN       NaN    NaN         NaN         NaN         NaN   \n",
       "129284       NaN       NaN    NaN         NaN         NaN         NaN   \n",
       "160554         A         N      A         NaN         NaN         NaN   \n",
       "\n",
       "        S_UNIT_COM  S_UNIT_RC S_EXT_FIN S_EXT_CND  U_BASE_FLOOR  U_NUM_PARK  \\\n",
       "131764         NaN        NaN       NaN       NaN           NaN         NaN   \n",
       "129284         NaN        NaN       NaN       NaN           NaN         NaN   \n",
       "160554         NaN        NaN       NaN       NaN           NaN         NaN   \n",
       "\n",
       "       U_CORNER U_ORIENT U_TOT_RMS  U_BDRMS  U_FULL_BTH  U_HALF_BTH  \\\n",
       "131764      NaN      NaN       NaN      NaN         NaN         NaN   \n",
       "129284      NaN      NaN       NaN      NaN         NaN         NaN   \n",
       "160554      NaN      NaN       NaN      NaN         NaN         NaN   \n",
       "\n",
       "       U_BTH_STYLE U_BTH_STYLE2 U_BTH_STYLE3 U_KITCH_TYPE U_KITCH_STYLE  \\\n",
       "131764         NaN          NaN          NaN          NaN           NaN   \n",
       "129284         NaN          NaN          NaN          NaN           NaN   \n",
       "160554         NaN          NaN          NaN          NaN           NaN   \n",
       "\n",
       "       U_HEAT_TYP U_AC  U_FPLACE U_INT_FIN U_INT_CND U_VIEW  \n",
       "131764        NaN  NaN       NaN       NaN       NaN    NaN  \n",
       "129284        NaN  NaN       NaN       NaN       NaN    NaN  \n",
       "160554        NaN  NaN       NaN       NaN       NaN    NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fix zip codes by prepending zero\n",
    "# TODO: also fix mailing zipcode\n",
    "shortFlag = prop['ZIPCODE'].str.len()==4\n",
    "shortZips = prop.loc[shortFlag, 'ZIPCODE']\n",
    "prop.loc[shortFlag, 'ZIPCODE'] = '0' + shortZips\n",
    "print('zip codes after prepending leading zero:')\n",
    "display(prop['ZIPCODE'].head(10))\n",
    "display(prop.sample(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "style fields that indicate the presence of luxury finishes:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>R_BTH_STYLE</th>\n",
       "      <th>R_BTH_STYLE2</th>\n",
       "      <th>R_BTH_STYLE3</th>\n",
       "      <th>R_KITCH_STYLE</th>\n",
       "      <th>R_KITCH_STYLE2</th>\n",
       "      <th>R_KITCH_STYLE3</th>\n",
       "      <th>U_BTH_STYLE</th>\n",
       "      <th>U_BTH_STYLE2</th>\n",
       "      <th>U_BTH_STYLE3</th>\n",
       "      <th>U_KITCH_STYLE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>L</td>\n",
       "      <td>M</td>\n",
       "      <td>M</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>L</td>\n",
       "      <td>L</td>\n",
       "      <td>L</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  R_BTH_STYLE R_BTH_STYLE2 R_BTH_STYLE3 R_KITCH_STYLE R_KITCH_STYLE2  \\\n",
       "0         NaN          NaN          NaN           NaN            NaN   \n",
       "1         NaN          NaN          NaN           NaN            NaN   \n",
       "2         NaN          NaN          NaN           NaN            NaN   \n",
       "3         NaN          NaN          NaN           NaN            NaN   \n",
       "4         NaN          NaN          NaN           NaN            NaN   \n",
       "5         NaN          NaN          NaN           NaN            NaN   \n",
       "6         NaN          NaN          NaN           NaN            NaN   \n",
       "7         NaN          NaN          NaN           NaN            NaN   \n",
       "8         NaN          NaN          NaN           NaN            NaN   \n",
       "9         NaN          NaN          NaN           NaN            NaN   \n",
       "\n",
       "  R_KITCH_STYLE3 U_BTH_STYLE U_BTH_STYLE2 U_BTH_STYLE3 U_KITCH_STYLE  \n",
       "0            NaN           M            M          NaN             M  \n",
       "1            NaN           S          NaN          NaN             S  \n",
       "2            NaN           M            S          NaN             M  \n",
       "3            NaN           N          NaN          NaN             N  \n",
       "4            NaN           L            M            M             L  \n",
       "5            NaN         NaN          NaN          NaN           NaN  \n",
       "6            NaN           S          NaN          NaN             S  \n",
       "7            NaN           M          NaN          NaN             M  \n",
       "8            NaN           L            L            L             L  \n",
       "9            NaN           M          NaN          NaN             M  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of luxury finished rooms:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    0\n",
       "4    2\n",
       "5    0\n",
       "6    0\n",
       "7    0\n",
       "8    4\n",
       "9    0\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# count number of luxury features\n",
    "luxCols = list(propDtypes.loc[propDtypes['is_luxury']].index)\n",
    "print('style fields that indicate the presence of luxury finishes:')\n",
    "luxVals = prop[luxCols]\n",
    "display(luxVals.head(10))\n",
    "# Count number of \"L\" values\n",
    "isLux = luxVals == 'L'\n",
    "nLux = isLux.sum(axis=1)\n",
    "print('# of luxury finished rooms:')\n",
    "display(nLux.head(10))\n",
    "prop['n_lux'] = nLux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "174317             Brighton\n",
       "59772                Boston\n",
       "164345             Brighton\n",
       "115695    Dorchester Center\n",
       "170494             Brighton\n",
       "Name: Neighborhood, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# merge zipcodes with neighborhood, which appears as last column\n",
    "prop = pd.merge(prop, zipNbh, left_on='ZIPCODE', right_index=True, how='left')\n",
    "display(prop['Neighborhood'].sample(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "South Boston                14312\n",
       "Jamaica Plain               11688\n",
       "Brighton                    11517\n",
       "Dorchester Center           10964\n",
       "West Roxbury                10936\n",
       "Boston Back Bay              9392\n",
       "Hyde Park                    9230\n",
       "Roslindale                   8987\n",
       "East Boston                  8722\n",
       "Boston South End             8706\n",
       "Dorchester Uphams Corner     7580\n",
       "Charlestown                  7171\n",
       "Roxbury                      6091\n",
       "Dorchester                   6091\n",
       "Boston Fenway                5209\n",
       "Boston West End              5209\n",
       "Mattapan                     4936\n",
       "Allston                      4437\n",
       "Dorchester Grove Hall        4304\n",
       "Boston North End             4201\n",
       "Boston Kenmore               3578\n",
       "Boston Chinatown             2859\n",
       "Boston Downtown Crossing     2152\n",
       "Boston Beacon  Hill          2130\n",
       "Roxbury Mission Hill         1891\n",
       "Boston                       1305\n",
       "Brookline                    1055\n",
       "Name: Neighborhood, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# show total properties by neighborhood\n",
    "allCnts = prop['Neighborhood'].value_counts()\n",
    "display(allCnts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter to residential properties\n",
    "# residential land use codes from data dictionary\n",
    "# exclude condo main entries, only want condos themselves\n",
    "resi_lu_codes = ['A','CD','R1','R2','R3','R4','RC','RL']\n",
    "# create a copy of underlying df just in case\n",
    "resi = prop.loc[prop['LU'].isin(resi_lu_codes)].copy()\n",
    "# residential count by nbhd\n",
    "resiCnts = resi['Neighborhood'].value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>all</th>\n",
       "      <th>residential</th>\n",
       "      <th>Pct_Residential</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Brookline</th>\n",
       "      <td>1055.0</td>\n",
       "      <td>1027.0</td>\n",
       "      <td>0.973460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>West Roxbury</th>\n",
       "      <td>10936.0</td>\n",
       "      <td>9855.0</td>\n",
       "      <td>0.901152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brighton</th>\n",
       "      <td>11517.0</td>\n",
       "      <td>10269.0</td>\n",
       "      <td>0.891638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hyde Park</th>\n",
       "      <td>9230.0</td>\n",
       "      <td>8137.0</td>\n",
       "      <td>0.881582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dorchester Center</th>\n",
       "      <td>10964.0</td>\n",
       "      <td>9590.0</td>\n",
       "      <td>0.874681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Roslindale</th>\n",
       "      <td>8987.0</td>\n",
       "      <td>7799.0</td>\n",
       "      <td>0.867809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Allston</th>\n",
       "      <td>4437.0</td>\n",
       "      <td>3821.0</td>\n",
       "      <td>0.861167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mattapan</th>\n",
       "      <td>4936.0</td>\n",
       "      <td>4241.0</td>\n",
       "      <td>0.859198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dorchester Grove Hall</th>\n",
       "      <td>4304.0</td>\n",
       "      <td>3648.0</td>\n",
       "      <td>0.847584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dorchester</th>\n",
       "      <td>6091.0</td>\n",
       "      <td>5057.0</td>\n",
       "      <td>0.830241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>East Boston</th>\n",
       "      <td>8722.0</td>\n",
       "      <td>7219.0</td>\n",
       "      <td>0.827677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dorchester Uphams Corner</th>\n",
       "      <td>7580.0</td>\n",
       "      <td>6186.0</td>\n",
       "      <td>0.816095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TOTAL CITY</th>\n",
       "      <td>174653.0</td>\n",
       "      <td>140832.0</td>\n",
       "      <td>0.806353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jamaica Plain</th>\n",
       "      <td>11688.0</td>\n",
       "      <td>9418.0</td>\n",
       "      <td>0.805784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston Back Bay</th>\n",
       "      <td>9392.0</td>\n",
       "      <td>7531.0</td>\n",
       "      <td>0.801853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston North End</th>\n",
       "      <td>4201.0</td>\n",
       "      <td>3295.0</td>\n",
       "      <td>0.784337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Charlestown</th>\n",
       "      <td>7171.0</td>\n",
       "      <td>5419.0</td>\n",
       "      <td>0.755683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>South Boston</th>\n",
       "      <td>14312.0</td>\n",
       "      <td>10810.0</td>\n",
       "      <td>0.755310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Roxbury Mission Hill</th>\n",
       "      <td>1891.0</td>\n",
       "      <td>1428.0</td>\n",
       "      <td>0.755156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston Kenmore</th>\n",
       "      <td>3578.0</td>\n",
       "      <td>2658.0</td>\n",
       "      <td>0.742873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston Downtown Crossing</th>\n",
       "      <td>2152.0</td>\n",
       "      <td>1545.0</td>\n",
       "      <td>0.717937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Roxbury</th>\n",
       "      <td>6091.0</td>\n",
       "      <td>4324.0</td>\n",
       "      <td>0.709900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston Fenway</th>\n",
       "      <td>5209.0</td>\n",
       "      <td>3686.0</td>\n",
       "      <td>0.707621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston West End</th>\n",
       "      <td>5209.0</td>\n",
       "      <td>3686.0</td>\n",
       "      <td>0.707621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston Chinatown</th>\n",
       "      <td>2859.0</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>0.696048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston Beacon  Hill</th>\n",
       "      <td>2130.0</td>\n",
       "      <td>1459.0</td>\n",
       "      <td>0.684977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston South End</th>\n",
       "      <td>8706.0</td>\n",
       "      <td>5944.0</td>\n",
       "      <td>0.682748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boston</th>\n",
       "      <td>1305.0</td>\n",
       "      <td>790.0</td>\n",
       "      <td>0.605364</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               all  residential  Pct_Residential\n",
       "Brookline                   1055.0       1027.0         0.973460\n",
       "West Roxbury               10936.0       9855.0         0.901152\n",
       "Brighton                   11517.0      10269.0         0.891638\n",
       "Hyde Park                   9230.0       8137.0         0.881582\n",
       "Dorchester Center          10964.0       9590.0         0.874681\n",
       "Roslindale                  8987.0       7799.0         0.867809\n",
       "Allston                     4437.0       3821.0         0.861167\n",
       "Mattapan                    4936.0       4241.0         0.859198\n",
       "Dorchester Grove Hall       4304.0       3648.0         0.847584\n",
       "Dorchester                  6091.0       5057.0         0.830241\n",
       "East Boston                 8722.0       7219.0         0.827677\n",
       "Dorchester Uphams Corner    7580.0       6186.0         0.816095\n",
       "TOTAL CITY                174653.0     140832.0         0.806353\n",
       "Jamaica Plain              11688.0       9418.0         0.805784\n",
       "Boston Back Bay             9392.0       7531.0         0.801853\n",
       "Boston North End            4201.0       3295.0         0.784337\n",
       "Charlestown                 7171.0       5419.0         0.755683\n",
       "South Boston               14312.0      10810.0         0.755310\n",
       "Roxbury Mission Hill        1891.0       1428.0         0.755156\n",
       "Boston Kenmore              3578.0       2658.0         0.742873\n",
       "Boston Downtown Crossing    2152.0       1545.0         0.717937\n",
       "Roxbury                     6091.0       4324.0         0.709900\n",
       "Boston Fenway               5209.0       3686.0         0.707621\n",
       "Boston West End             5209.0       3686.0         0.707621\n",
       "Boston Chinatown            2859.0       1990.0         0.696048\n",
       "Boston Beacon  Hill         2130.0       1459.0         0.684977\n",
       "Boston South End            8706.0       5944.0         0.682748\n",
       "Boston                      1305.0        790.0         0.605364"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "compDf = pd.concat([allCnts.to_frame(), resiCnts.to_frame()], axis=1)\n",
    "compDf.columns = ['all', 'residential']\n",
    "compDf.loc['TOTAL CITY', :] = [allCnts.sum(), resiCnts.sum()]\n",
    "compDf['Pct_Residential'] = compDf['residential'] / compDf['all']\n",
    "compDf = compDf.sort_values('Pct_Residential', ascending=False)\n",
    "display(compDf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Brookline and West Roxbury are the most residential neighborhoods in Boston."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting Condo Values: More Data Prep\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter to residential condo units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter to just condo units for regressions\n",
    "condos = resi.loc[resi['LU'] == 'CD']\n",
    "# index condos by PID, but keep column around\n",
    "condos = condos.set_index('PID', drop=False)\n",
    "# only one building seems to be missing CM_ID, and it's also missing other attributes like age,\n",
    "#  so it's probably OK to drop it.\n",
    "condos = condos.loc[~condos['CM_ID'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop columns related to single family homes\n",
    "condos = condos.drop(columns=['R_BLDG_STYL', 'R_ROOF_TYP', 'R_EXT_FIN',\n",
    "       'R_TOTAL_RMS', 'R_BDRMS', 'R_FULL_BTH', 'R_HALF_BTH', 'R_BTH_STYLE',\n",
    "       'R_BTH_STYLE2', 'R_BTH_STYLE3', 'R_KITCH', 'R_KITCH_STYLE',\n",
    "       'R_KITCH_STYLE2', 'R_KITCH_STYLE3', 'R_HEAT_TYP', 'R_AC', 'R_FPLACE',\n",
    "       'R_EXT_CND', 'R_OVRALL_CND', 'R_INT_CND', 'R_INT_FIN', 'R_VIEW'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Join with Condo Building data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract condo main info to join with units\n",
    "condoMain = prop.loc[prop['LU'] == 'CM']\n",
    "condoMain.head()\n",
    "condoMain.set_index('CM_ID', drop=False, inplace=True)\n",
    "# remove duplicate condo main entries\n",
    "condoMain = condoMain.loc[~condoMain.index.duplicated(keep='first')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['S_NUM_BLDG', 'S_BLDG_STYL', 'S_UNIT_RES', 'S_EXT_FIN', 'S_EXT_CND']"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# join S_ columns with CM\n",
    "cmCols = condoMain.columns[condoMain.columns.str.startswith('S_')]\n",
    "# exclude certain columns we won't use related to mixed-use buildings and units\n",
    "cmExcludeCols = {'S_UNIT_COM', 'S_UNIT_RC'}\n",
    "cmCols = [cmc for cmc in cmCols if (cmc not in cmExcludeCols)]\n",
    "cmCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmData = pd.merge(condos[['PID', 'CM_ID']], condoMain[['CM_ID'] + cmCols], on='CM_ID', \n",
    "                  how='inner')\n",
    "# index joined data by PID for use in fillna\n",
    "cmData = cmData.set_index('PID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fix null or zero building number with 1\n",
    "flagNb = cmData['S_NUM_BLDG'].isnull() | (cmData['S_NUM_BLDG'] == 0)\n",
    "cmData.loc[flagNb, 'S_NUM_BLDG'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fill missing condo main properties where possible, e.g. residential unit count\n",
    "# hacky procedure, could be generalized though, relies on a group col (CM_ID)\n",
    "fillCols = ['S_UNIT_RES']\n",
    "for fc in fillCols:\n",
    "    fillFlag = cmData[fc].isnull() | (cmData[fc] == 0)\n",
    "    fillVal = cmData.join(cmData.groupby('CM_ID')[fc].size(), on='CM_ID', rsuffix='_r')\n",
    "    cmData.loc[fillFlag, fc] = fillVal.loc[fillFlag, '{}_r'.format(fc)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loop column-wise to fill missing values in condo units\n",
    "for cmC in cmCols:\n",
    "    condos[cmC] = condos[cmC].fillna(cmData[cmC])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add calculated column indicating multi-building complex\n",
    "condos['is_in_complex'] = condos['S_NUM_BLDG'] > 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>S_NUM_BLDG</th>\n",
       "      <th>S_BLDG_STYL</th>\n",
       "      <th>S_UNIT_RES</th>\n",
       "      <th>S_EXT_FIN</th>\n",
       "      <th>S_EXT_CND</th>\n",
       "      <th>is_in_complex</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>603773008</th>\n",
       "      <td>1.0</td>\n",
       "      <td>FS</td>\n",
       "      <td>8.0</td>\n",
       "      <td>C</td>\n",
       "      <td>G</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604143006</th>\n",
       "      <td>1.0</td>\n",
       "      <td>RE</td>\n",
       "      <td>3.0</td>\n",
       "      <td>M</td>\n",
       "      <td>A</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>702692002</th>\n",
       "      <td>1.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>3.0</td>\n",
       "      <td>W</td>\n",
       "      <td>A</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1503026002</th>\n",
       "      <td>1.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>3.0</td>\n",
       "      <td>F</td>\n",
       "      <td>G</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503196002</th>\n",
       "      <td>1.0</td>\n",
       "      <td>RM</td>\n",
       "      <td>5.0</td>\n",
       "      <td>B</td>\n",
       "      <td>G</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            S_NUM_BLDG S_BLDG_STYL  S_UNIT_RES S_EXT_FIN S_EXT_CND  \\\n",
       "PID                                                                  \n",
       "603773008          1.0          FS         8.0         C         G   \n",
       "604143006          1.0          RE         3.0         M         A   \n",
       "702692002          1.0          DK         3.0         W         A   \n",
       "1503026002         1.0          DK         3.0         F         G   \n",
       "503196002          1.0          RM         5.0         B         G   \n",
       "\n",
       "            is_in_complex  \n",
       "PID                        \n",
       "603773008           False  \n",
       "604143006           False  \n",
       "702692002           False  \n",
       "1503026002          False  \n",
       "503196002           False  "
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now most condos will have at least some of the condo building properties available\n",
    "condos[cmCols + ['is_in_complex']].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Condo Age Column and Filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    63853.000000\n",
       "mean      1991.380045\n",
       "std         23.822814\n",
       "min       1500.000000\n",
       "25%       1984.000000\n",
       "50%       1997.000000\n",
       "75%       2006.000000\n",
       "max       2018.000000\n",
       "Name: YR_REMOD, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# get latest construction year for each condo property (new or remodel)\n",
    "# fill in empty remodels with year built so that there are no NaN values\n",
    "condos.loc[(condos['YR_REMOD'] == 0), 'YR_REMOD'] = np.nan\n",
    "condos['YR_REMOD'].fillna(condos['YR_BUILT'], inplace=True)\n",
    "# in case year built was also 0, flag as nan again\n",
    "condos.loc[(condos['YR_REMOD'] == 0), 'YR_REMOD'] = np.nan\n",
    "condos = condos.loc[~condos['YR_REMOD'].isnull()]\n",
    "display(condos['YR_REMOD'].describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    63853.000000\n",
       "mean        27.619955\n",
       "std         23.822814\n",
       "min          1.000000\n",
       "25%         13.000000\n",
       "50%         22.000000\n",
       "75%         35.000000\n",
       "max        519.000000\n",
       "Name: age, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# calculate years since remodel\n",
    "condos['age'] = dt.datetime.now().year - condos['YR_REMOD']\n",
    "display(condos['age'].describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter to condos with age <= 250 years to reduce the effect of age outliers\n",
    "condos = condos.loc[condos['age'] <= 250]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attribute-based outlier removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# exclude condo units with > 7 bedrooms (only 1 outlier)\n",
    "condos = condos.loc[condos['U_BDRMS'] <= 7]\n",
    "# limit to condo units with 4 or fewer full bathrooms\n",
    "condos = condos.loc[condos['U_FULL_BTH'] <= 4]\n",
    "# filter based on # of parking spaces, to eliminate outliers (3 units >= 6 spaces)\n",
    "condos = condos.loc[condos['U_NUM_PARK'] < 6]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter Assessed Value Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count       63812\n",
       "mean       674784\n",
       "std        714466\n",
       "min         41700\n",
       "25%        344500\n",
       "50%        488300\n",
       "75%        722800\n",
       "max      34057908\n",
       "Name: AV_TOTAL, dtype: int32"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Count')"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEPCAYAAACQmrmQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGZpJREFUeJzt3X2UXHWd5/F3dwgkSgIYmgWGB5eBfMGHISwPzsiDjHLcZYRBjgMIGVh2JMgB1N0DOromh8EDOjsuEXEMzAQjnIkIs7DAQcgsKzoS5cFBBWZAvosrxEXikgmsSRgCSTr7x70Flba7q7rTv+qq5P06JydV33vr1rd+J51P33vr/m7f5s2bkSSphP7JbkCStO0yZCRJxRgykqRiDBlJUjGGjCSpGENGklSMISNJKsaQkSQVY8hIkooxZCRJxRgykqRidpjsBibBTsCRwEpg0yT3Ikm9YgqwF/APwKvtvmh7DJkjgeWT3YQk9ahjge+3u/L2GDIrAV566WUGB8c+A/WsWTuzevW6CW+qJHsur9f6BXvulF7reaR++/v72G23N0P9f2i7tseQ2QQwOLh5XCHTeG2vsefyeq1fsOdO6bWeW/Q7ptMMnviXJBVjyEiSijFkJEnFGDKSpGIMGUlSMYaMJKkYQ0aSVMz2eJ3MhJoxczrTdqqGcf2rG1m75pVJ7kiSuochs5Wm7bQDJ19yJwB3XXUKaye5H0nqJh4ukyQVY8hIkooxZCRJxRgykqRiDBlJUjGGjCSpGENGklSMISNJKsaQkSQVY8hIkooxZCRJxRgykqRiik6QGRGXAafXT+/OzE9FxNeBY4CX6/rlmXl7RJwALASmA7dk5vx6G3OA64GZwP3ABZm5MSL2A5YCewAJzM3MdSU/jyRpbIrtydSh8X7gMGAOcHhEnAocARyXmXPqP7dHxHRgCXAKcAhwZEScWG9qKXBxZs4G+oB5dX0RsCgzDwYeARaU+iySpPEpebhsJXBJZr6WmRuAnwL71X+WRMTjEXF5RPQDRwFPZ+YzmbmRKlhOi4j9gemZ+VC9zRvq+lTgOODW5nrBzyJJGodih8sy84nG44g4iOqw2bHA8cCFwK+BbwEfAdZRhVLDSmAfYO8R6rsDa+pAaq63bdasncey+hYGBmaMa9lk6ta+RtNrPfdav2DPndJrPU9kv8VvWhYRbwfuBj6ZmQmc2rTsK8A5VHskm5te1gcMUu1ptVOnrrdt9ep1DA4O3URrAwMzWLVq7RbPmzUv6xZDe+4FvdZzr/UL9twpvdbzSP329/eN65fzot8ui4ijgfuAT2fmjRHxzoj4UNMqfcAG4Dlgr6b6nsDzo9RfAHaJiCl1fa+6LknqIiVP/O8L3AGclZk31+U+4OqI2K0+r3I+cDvwcPWSOLAOjrOAZZm5AlhfhxXA2XV9A7AcOKOunwMsK/VZJEnjU/Jw2aXANGBhRDRq1wFfAH4ATAVuy8xvAkTEucBt9Wvu4Y2T+nOBxRExE/gxcE1dvxC4MSLmA78Aziz4WSRJ41DyxP8ngE+MsHjRMOvfBxw6TP0xqm+fDa2voPoSgSSpS3nFvySpGENGklSMISNJKsaQkSQVY8hIkooxZCRJxRgykqRiDBlJUjGGjCSpGENGklSMISNJKsaQkSQVY8hIkooxZCRJxRgykqRiDBlJUjGGjCSpGENGklSMISNJKsaQkSQVY8hIkooxZCRJxRgykqRiDBlJUjGGjCSpGENGklSMISNJKsaQkSQVs8NkN9CLZsyczrSdHDpJaqXo/5QRcRlwev307sz8VEScACwEpgO3ZOb8et05wPXATOB+4ILM3BgR+wFLgT2ABOZm5rqI2BX4BnAAsAo4PTN/VfLzNEzbaQdOvuROAO666pROvKUk9aRih8vqMHk/cBgwBzg8Is4ElgCnAIcAR0bEifVLlgIXZ+ZsoA+YV9cXAYsy82DgEWBBXb8CWJ6ZhwCLgS+X+iySpPEpeU5mJXBJZr6WmRuAnwKzgacz85nM3EgVLKdFxP7A9Mx8qH7tDXV9KnAccGtzvX78Aao9GYBvAifW60uSukSxw2WZ+UTjcUQcRHXY7CtU4dOwEtgH2HuE+u7AmjqQmus0v6Y+rLYGGACeb6e/WbN2HuMnas/AwIwi291a3drXaHqt517rF+y5U3qt54nst/jZ64h4O3A38ElgI9XeTEMfMEi1R7W5jTp1vbFOs76mZS2tXr2OwcGhm26t1eCvWrV2zNssbWBgRlf2NZpe67nX+gV77pRe63mkfvv7+8b1y3nRrzBHxNHAfcCnM/NG4Dlgr6ZV9qTa8xip/gKwS0RMqet78caeyi/r9YiIHYAZwOoyn0SSNB4lT/zvC9wBnJWZN9flh6tFcWAdHGcByzJzBbC+DiWAs+v6BmA5cEZdPwdYVj++p35OvXx5vb4kqUuUPFx2KTANWBgRjdp1wLnAbfWye3jjpP5cYHFEzAR+DFxT1y8EboyI+cAvgDPr+gLghoh4Avh/9eslSV2k5In/TwCfGGHxocOs/xhw1DD1FcDxw9RfBP5w67qUJJXktDKSpGIMGUlSMYaMJKkYQ0aSVIwhI0kqxpCRJBVjyEiSijFkJEnFGDKSpGIMGUlSMYaMJKkYQ0aSVIwhI0kqxpCRJBVjyEiSijFkJEnFGDKSpGIMGUlSMYaMJKkYQ0aSVIwhI0kqxpCRJBVjyEiSijFkJEnFtBUyEfG1YWq3Tnw7kqRtyQ6jLYyIa4HfAo6NiIGmRVOBA0o2JknqfaOGDPA14B3AocBtTfWNwEOlmpIkbRtGDZnMfAR4JCK+nZnPdagnSdI2otWeTMO+EfE3wFuAvkYxM3+nSFeSpG1CuyHzV8ANwI+BzWN5g4iYCTwAnJSZz0bE14FjgJfrVS7PzNsj4gRgITAduCUz59evnwNcD8wE7gcuyMyNEbEfsBTYA0hgbmauG0tvkqSy2g2ZjZm5cKwbj4h3AYuB2U3lI4DjMnNl03rTgSXAe4D/A9wdESdm5jKqIDkvMx+qv+U2D7gWWAQsysybI2IBsAD407H2KEkqp93rZP4pIt45ju3PAy4CngeIiDcB+wFLIuLxiLg8IvqBo4CnM/OZzNxIFSynRcT+wPTMbHzJ4Ia6PhU4Dri1uT6O/iRJBbW7J3MA8KOIWAG80ii2OieTmecBRESjtCfwHeBC4NfAt4CPAOuAlU0vXQnsA+w9Qn13YE0dSM31ts2atfNYVm/bwMCMItvdWt3a12h6rede6xfsuVN6reeJ7LfdkPnsRLxZZv4cOLXxPCK+ApxDtUfSfK6nDxik2tNqp05db9vq1esYHBzT6SWg9eCvWrV2zNssbWBgRlf2NZpe67nX+gV77pRe63mkfvv7+8b1y3m7IfOPY97yMOpDbrMzs3HNTR+wAXgO2Ktp1T2pDrGNVH8B2CUipmTmpnqd5yeiR0nSxGn3nMw/A6ua/l4FPD6O9+sDro6I3erzKucDtwMPAxERB0bEFOAsYFlmrgDWR8TR9evPrusbgOXAGXX9HGDZOPqRJBXU1p5MZr4eRhGxI1UIxMivGHE7j0fEF4AfUE1Nc1tmfrPe7rlUswpMA+7hjZP6c4HF9VehfwxcU9cvBG6MiPnAL4Azx9qPJKmsdg+XvS4zXwNuiIhHgM+0+Zq3Nj1eRPX146Hr3Ec1fc3Q+mNU3z4bWl8BHN9u35KkzmsrZCLiLU1P+6iuddmtSEeSpG1Gu3sy/0z1ba7GlDIvAB8v0pEkaZsx5nMykiS1q93DZf3ApcCJVCfs7wU+33QxpCRJv6HdPZQvAO8Fvkw1ieW7gS+WakqStG1o95zMvwOOqK9PISLuBh4D/lOpxiRJva/dPZn+RsAAZOarVFfqS5I0onb3ZB6NiC8Bf0n1LbOPMb4r/iVJ25F292Quorou5gGqKWB2pwoaSZJGNOqeTD2FzGLgjsw8t67dDWwC1hTvTpLU01rtyXyO6rbHP2iqzQN2Bf6sUE+SpG1Eq5A5CTgrM19oFDLzeapZj08d8VWSJNE6ZF7LzFeGFjNzDfBqmZYkSduKViGzKSJ+41aQdW1qmZYkSduKViHzTeD6iHhzo1A/vp7q3i+SJI2o1XUyVwPXAb+KiCeoQukQ4BtUXwqQJGlEo4ZMZg4C50fElcDhwCDwcGau7ERzkqTe1u5U/yuAFYV7kSRtY7xPjCSpGENGklSMISNJKsaQkSQVY8hIkooxZCRJxRgykqRiDBlJUjGGjCSpGENGklSMISNJKqatucvGKyJmAg8AJ2XmsxFxArAQmA7ckpnz6/XmUN0+YCZwP3BBZm6MiP2ApcAeQAJzM3NdROxKNRP0AcAq4PTM/FXJzyJJGrtiezIR8S7g+8Ds+vl0YAlwCtXtAo6MiBPr1ZcCF2fmbKAPmFfXFwGLMvNg4BFgQV2/AliemYcAi4Evl/ockqTxK3m4bB5wEfB8/fwo4OnMfCYzN1IFy2kRsT8wPTMfqte7oa5PBY4Dbm2u148/QLUnA9WN1U6s15ckdZFih8sy8zyAiGiU9gaa70OzEthnlPruwJo6kJrrW2yrPqy2BhjgjUBradasncfwado3MPAbd6vuCt3a12h6rede6xfsuVN6reeJ7LfoOZkh+oHNTc/7qG6C1m6dut5Yp1lf07K2rF69jsHBoZtvrdXgr1q1dszbLG1gYEZX9jWaXuu51/oFe+6UXut5pH77+/vG9ct5J79d9hywV9PzPan2PEaqvwDsEhFT6vpevLGn8st6PSJiB2AGsLpY55KkcelkyDwMREQcWAfHWcCy+q6b6yPi6Hq9s+v6BmA5cEZdPwdYVj++p35OvXx5vb4kqYt0LGQycz1wLnAb8CTwFG+c1J8LfCkingJ2Bq6p6xcC50fEk8CxwPy6vgD43Yh4ol7nok58BknS2BQ/J5OZb216fB9w6DDrPEb17bOh9RXA8cPUXwT+cCL7lCRNPK/4lyQVY8hIkooxZCRJxRgykqRiDBlJUjGGjCSpGENGklSMISNJKsaQkSQV08lZmLd5r23Y9Poszetf3cjaNa9MckeSNLkMmQm049QpnHzJnQDcddUp9M7k3pJUhofLJEnFGDKSpGIMGUlSMYaMJKkYQ0aSVIwhI0kqxpCRJBVjyEiSijFkJEnFGDKSpGIMGUlSMYaMJKkYQ0aSVIwhI0kqxpCRJBVjyEiSijFkJEnFGDKSpGIm5fbLEfFdYA9gQ136KPDbwHxgKnB1Zn61XvcEYCEwHbglM+fX9TnA9cBM4H7ggszc2MnPIUkaXcf3ZCKiD5gNHJqZczJzDvAccCVwDDAHOD8i3hYR04ElwCnAIcCREXFivamlwMWZORvoA+Z1+KNIklqYjD2ZqP++NyJmAYuBtcB3MvNFgIi4Ffgj4HvA05n5TF1fCpwWEU8C0zPzoXpbNwCXA9d27FNIklqajJDZDbgP+BjVobG/B24BVjatsxI4Cth7mPo+o9TbNmvWzmNse+wGBmYUf492dVMv7eq1nnutX7DnTum1niey346HTGY+CDzYeB4RX6M653JF02p9wCDV4bzNY6i3bfXqdQwObm694hBjGfxVq9aOefslDAzM6Jpe2tVrPfdav2DPndJrPY/Ub39/37h+OZ+MczLHRMT7mkp9wLPAXk21PYHnqc7VjKUuSeoik/EV5l2BL0bEtIiYAfx74I+B90XEQES8CfgQ8HfAw0BExIERMQU4C1iWmSuA9RFxdL3Ns4FlHf8kkqRRdTxkMvNbwN3AT4AfAUsy8wfAZ4HvAo8CN2XmDzNzPXAucBvwJPAUcGu9qbnAlyLiKWBn4JpOfg5JUmuTcp1MZi4AFgyp3QTcNMy69wGHDlN/jOrLAZKkLuUV/5KkYgwZSVIxhowkqRhDRpJUjCEjSSrGkJEkFWPISJKKMWQkScUYMpKkYgwZSVIxkzKtzPbgtQ2bXr8twPpXN7J2zSuT3JEkdZ4hU8iOU6dw8iV3AnDXVafQO3eTkKSJ4+EySVIxhowkqRhDRpJUjCEjSSrGkJEkFWPISJKKMWQkScUYMpKkYrwYswO8+l/S9sqQ6QCv/pe0vfJwmSSpGENGklSMISNJKsZzMh3mlwAkbU8MmQ7zSwCStieGzCRyr0bSts6QmUTu1Uja1vV0yETEWcB8YCpwdWZ+dZJbGjf3aiRti3r222UR8VvAlcAxwBzg/Ih42+R2NX6NvZqTL7mT/v4+BgZmMDAwgxkzp092a5I0br28J3MC8J3MfBEgIm4F/gj4XIvXTQHo7+/bqjffY7fpxR7vOHUKH7niXgCu/dP3vb6H8+prm9hpxym/+fjVjaxbt37Ufrf2806GXuu51/oFe+6UXut5uH6balPGsq2+zZs3T0BLnRcRnwHenJnz6+fnAUdl5vktXnoMsLx0f5K0jToW+H67K/fynkw/0JyQfcBgG6/7B6pBWglsKtCXJG2LpgB7Uf0f2rZeDpnnqMKiYU/g+TZe9ypjSGFJ0uv+91hf0Msh823gzyJiAHgZ+BDQ6lCZJKmDevbbZZn5S+CzwHeBR4GbMvOHk9uVJKlZz574lyR1v57dk5EkdT9DRpJUjCEjSSrGkJEkFdPLX2EuqtXkmxExB7gemAncD1yQmRs73uiWPbXq+TLgT4CX6tLiyZ5UNCJmAg8AJ2Xms0OWdd0YQ8ueu3GMLwNOr5/enZmfGrK868a5jZ67apwj4nNU01ptBr6WmQuHLO/GMW7V84SMsXsyw2hz8s2lwMWZOZtqtoF5ne1yS232fATw4cycU/+Z7P/83kV1YezsEVbpqjGGtnrutjE+AXg/cBjVv4vDI+LUIat11Ti32XPXjHNEvAd4L/A7dV8fi4gYslq3jXE7PU/IGBsyw3t98s3MfBloTL4JQETsD0zPzIfq0g3AaR3vckuj9lw7AvjPEfF4RPxlREzreJdbmgdcxDAzNXTpGMMoPde6bYxXApdk5muZuQH4KbBfY2GXjvOoPde6Zpwz83vA79d7JntQHSF6ubG8G8e4Vc+1CRljQ2Z4e1P9Q29YCewzhuWTYdSeImJn4CfAJ4F/A+wKLOhkg0Nl5nmZOdJkpd04xqP23KVj/ETjP7eIOIjqENQ9Tat03Ti36rlLx3lDRFwOPAncB/yyaXHXjTGM3vNEjrEhM7xWk2+Od3LOkkbtKTPXZeYfZOZT9W8vVwF/0OEex6Ibx3hU3TzGEfF24H8Cn8zMp5sWde04j9Rzt45zZl4GDAD7suXhsK4d45F6nsgxNmSG9xzVbKMNQyffbLV8MozaU0TsFxF/0rS8D9jQod7GoxvHeFTdOsYRcTTVb6qfzswbhyzuynEereduG+eIOLg+sU9m/gvw36nOdTR03Ri36nkix9iQGd63gfdFxEBEvIlq8s2/ayzMzBXA+voHAeBsYFnn29zCqD0DrwB/ERH/OiL6qM4r3D4JfbalS8e4la4b44jYF7gDOCszbx66vBvHuVXPdN84HwAsjoidImJH4BSaZnrvxjGmRc9M4BgbMsMYafLNiLgnIo6oV5sLfCkingJ2Bq6ZnG4rrXrOzFXAR4G7gKT6zeSqSWt4BN08xiPp8jG+FJgGLIyIR+s/F3T5OI/ac7eNc2beA9xNdQ7jR8ADmXlzN49xq54ncoydIFOSVIx7MpKkYgwZSVIxhowkqRhDRpJUjBNkStJ2YrTJXZvWmUM19U3DAPBSZr5jPO9pyEjDiIhrgOPqp28DnqG6dgDg9zLzlRFetxvw3zLzhBbbP4/qB/2DTbV+4OfApZl565D1rwX+JTMvGWWb3wf+a2beMeqH03apntx1MSNP7gpAZj5KNTEp9TV3PwQuGO/7GjLSMDLz443HEfEsMDczH2njpbOoJhYcz3sORsRfAR+hmuC08f5vBj4MHDme7Uq1xuSuf9MoRMQ5wH+kOnXyI+CizFzf9JrPAN/LzOYLNcfEkJHGoZ4q/S+oLhp8DfhsZt4LfB2YERGPZuaciJgHnAfsCLwFuDIz/3qUTV8PzI+IfTLzubr2YeDBzPxZPXHhdcCBVIH2a6rp2H/W1NuBwCOZuesIz8+nutCuH1hFNQX9/5qAYVEXy8zzABoz+tdzw80D3p2Z6yPiC1QXwl5RL98FOB9459a8ryf+pTGKiAHgb6l+6zuU6sZON0XEfsB/ANbWATOzXnZiZh5GddX3fxlt2/WV1rcD5zaVzwca9/L4ALAqM383Mw+iumL7ojH0/l7gTOCYuqeradpr0nbl94GDgIci4lGqqWUOblr+x8AdmfnC1ryJezLS2P0e8FTj8Flm/mNEPAy8B3iwsVJmromIk4GT6ynrD6OaUqSVrwJLI+JKqt8id6ee6yozb4mIn0XEx6n2Zo6jutNiu04CAngw3rhH1UBE7JKZvx7DdtT7pgB/2zg0XO8lN2fCB4HPb+2buCcjjd0Utpy6HaqfpanNhfpmVT+hunfIcqr7cfS12nhmPkh1GOx4qr2YazNzsN7mx4C/BtYB36Daoxq6zc1DajsO6f3rjbsdAocDRxow26W/B06NiD3qSTCvpTo/Q/38cJp+aRovQ0YauweAdzQmP4yIdwJHU/3QbgR2qH9Ij6S6QdXngXuBk2n/Z24R1SGzDwJLmur/FliSmUuAp6n2TKYMee1LwJvijV2VM5uW/Q9gbkT8q/r5RXVv2s5k5mPA5cB3gCeo/h39eb14AHhtyJcAxsXDZdIYZeb/jYgzgGujuiXtJuDszPx5REyl2nv5J+DdVOdokuomVd8FXoqI327jbW6iOn9zR2a+2FT/InBdRHyUam/lIbY8jk5mvhgRnwHujYgXaDrnkpn3RMRC4L6I2EwVSB8a+yioV2XmW5seX0/1ZZOh67xAdd+breYszJKkYjxcJkkqxpCRJBVjyEiSijFkJEnFGDKSpGIMGUlSMYaMJKkYQ0aSVMz/B8Bc6RtaTEneAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# examine total assessed value distribution\n",
    "display(condos['AV_TOTAL'].describe().astype(int))\n",
    "fig = plt.hist(condos['AV_TOTAL'], bins=100)\n",
    "plt.xlabel('Total Value')\n",
    "plt.ylabel('Count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8000     808400.00\n",
       "0.8500     944535.00\n",
       "0.9000    1212190.40\n",
       "0.9500    1783715.65\n",
       "0.9750    2471590.00\n",
       "0.9825    2877553.75\n",
       "0.9900    3660083.20\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# figure out what percentile to cut\n",
    "pctls = [0.8, 0.85, 0.9, 0.95, 0.975, 0.9825, 0.99]\n",
    "display(pd.Series({k: v for k, v in zip(pctls, condos['AV_TOTAL'].quantile(pctls))}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The histogram above shows that the distribution of condo values is highly right-skewed-- to limit the impact this has on our regressions, we must exclude some outliers.\n",
    "\n",
    "Based on the percentiles above, I have chosen to limit the condos in the sample to the 95th percentile of values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "cutPctl = 0.95\n",
    "condos = condos.loc[condos['AV_TOTAL'] <= condos['AV_TOTAL'].quantile(cutPctl)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create prediction df\n",
    "# pred = condos.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "# force number of rooms to be numeric\n",
    "# pred['U_TOT_RMS'] = pred['U_TOT_RMS'].astype(float, errors='ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Categorical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define categorical columns to include and transform via get_dummies\n",
    "catCols = ['ZIPCODE', 'OWN_OCC', 'S_BLDG_STYL',\n",
    "       'S_EXT_FIN', 'S_EXT_CND', 'U_CORNER',\n",
    "       'U_ORIENT', 'U_BTH_STYLE', 'U_BTH_STYLE2', 'U_BTH_STYLE3', 'U_KITCH_TYPE',\n",
    "       'U_KITCH_STYLE', 'U_HEAT_TYP', 'U_AC', 'U_INT_FIN',\n",
    "       'U_INT_CND', 'U_VIEW', 'Neighborhood', 'is_in_complex']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ensure consistent capitalization across values by putting everything into upper case\n",
    "for cc in catCols:\n",
    "    if condos[cc].dtype == 'object':\n",
    "        condos[cc] = condos[cc].str.upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ZIPCODE:\n",
      "['02108' '02128' '02129' '02114' '02113' '02109' '02115' '02210' '02110'\n",
      " '02116' '02111' '02118' '02215' '02127' '02122' '02124' '02125' '02119'\n",
      " '02120' '02130' '02199' '02131' '02121' '02126' '02136' '02135' '02132'\n",
      " '02467' '02134' '02446']\n",
      "object\n",
      "OWN_OCC:\n",
      "['Y' 'N']\n",
      "object\n",
      "S_BLDG_STYL:\n",
      "['RM' 'RE' 'DK' 'SD' 'TH' 'FS' 'CV' 'LR' 'DX' 'MR' 'HR' 'CM' nan]\n",
      "object\n",
      "S_EXT_FIN:\n",
      "['B' 'M' 'W' 'F' 'C' 'A' 'V' 'S' 'P' 'G' 'O' 'K' nan 'U']\n",
      "object\n",
      "S_EXT_CND:\n",
      "['G' 'A' 'E' 'F' 'P' nan]\n",
      "object\n",
      "U_CORNER:\n",
      "['N' 'Y']\n",
      "object\n",
      "U_ORIENT:\n",
      "['F' 'A' 'B' 'T' 'M' 'E' 'C']\n",
      "object\n",
      "U_BTH_STYLE:\n",
      "['M' 'S' 'N' 'L']\n",
      "object\n",
      "U_BTH_STYLE2:\n",
      "['M' nan 'S' 'N' 'L']\n",
      "object\n",
      "U_BTH_STYLE3:\n",
      "[nan 'S' 'M' 'L' 'N']\n",
      "object\n",
      "U_KITCH_TYPE:\n",
      "['O' 'P' 'F' 'N']\n",
      "object\n",
      "U_KITCH_STYLE:\n",
      "['M' 'S' 'N' 'L']\n",
      "object\n",
      "U_HEAT_TYP:\n",
      "['W' 'F' 'E' 'P' 'S' 'N' 'O']\n",
      "object\n",
      "U_AC:\n",
      "['N' 'C' 'D']\n",
      "object\n",
      "U_INT_FIN:\n",
      "['N' 'S' 'E']\n",
      "object\n",
      "U_INT_CND:\n",
      "['G' 'A' 'E' 'P' 'F']\n",
      "object\n",
      "U_VIEW:\n",
      "['A' 'G' 'F' 'E' 'P' 'S']\n",
      "object\n",
      "Neighborhood:\n",
      "['BOSTON BEACON  HILL' 'EAST BOSTON' 'CHARLESTOWN' 'BOSTON WEST END'\n",
      " 'BOSTON NORTH END' 'BOSTON FENWAY' 'BOSTON' 'BOSTON DOWNTOWN CROSSING'\n",
      " 'BOSTON BACK BAY' 'BOSTON CHINATOWN' 'BOSTON SOUTH END' 'BOSTON KENMORE'\n",
      " 'SOUTH BOSTON' 'DORCHESTER' 'DORCHESTER CENTER'\n",
      " 'DORCHESTER UPHAMS CORNER' 'ROXBURY' 'ROXBURY MISSION HILL'\n",
      " 'JAMAICA PLAIN' 'ROSLINDALE' 'DORCHESTER GROVE HALL' 'MATTAPAN'\n",
      " 'HYDE PARK' 'BRIGHTON' 'WEST ROXBURY' 'BROOKLINE' 'ALLSTON']\n",
      "object\n",
      "is_in_complex:\n",
      "[False  True]\n",
      "bool\n"
     ]
    }
   ],
   "source": [
    "# print values present for each categorical variable\n",
    "# for now, leave nan values in and encode missing values as its own group\n",
    "for cc in catCols:\n",
    "    print(f'{cc}:\\n{condos[cc].unique()}\\n{condos[cc].dtype}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of condos by neighborhood:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SOUTH BOSTON                6720\n",
       "BRIGHTON                    5823\n",
       "BOSTON BACK BAY             5124\n",
       "JAMAICA PLAIN               4800\n",
       "BOSTON SOUTH END            4503\n",
       "CHARLESTOWN                 3427\n",
       "BOSTON WEST END             3009\n",
       "BOSTON FENWAY               2870\n",
       "BOSTON NORTH END            2332\n",
       "BOSTON KENMORE              2323\n",
       "ALLSTON                     2092\n",
       "DORCHESTER UPHAMS CORNER    1856\n",
       "EAST BOSTON                 1837\n",
       "ROSLINDALE                  1784\n",
       "BOSTON CHINATOWN            1580\n",
       "DORCHESTER                  1470\n",
       "DORCHESTER CENTER           1465\n",
       "BOSTON DOWNTOWN CROSSING    1218\n",
       "WEST ROXBURY                1217\n",
       "ROXBURY                      983\n",
       "BOSTON BEACON  HILL          890\n",
       "BROOKLINE                    834\n",
       "HYDE PARK                    787\n",
       "BOSTON                       588\n",
       "ROXBURY MISSION HILL         436\n",
       "DORCHESTER GROVE HALL        384\n",
       "MATTAPAN                     269\n",
       "Name: Neighborhood, dtype: int64"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('# of condos by neighborhood:')\n",
    "condos['Neighborhood'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zip Code Demographic Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge with zipcode data\n",
    "search = SearchEngine(simple_zipcode=False)\n",
    "# identify columns of interest from full us zip code dataset\n",
    "zipCols = ['population', 'population_density', 'land_area_in_sqmi', 'water_area_in_sqmi', \n",
    "           'housing_units', 'occupied_housing_units', 'median_home_value', 'median_household_income']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "example zip code statistics to be merged with property data:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>population</th>\n",
       "      <th>population_density</th>\n",
       "      <th>land_area_in_sqmi</th>\n",
       "      <th>water_area_in_sqmi</th>\n",
       "      <th>housing_units</th>\n",
       "      <th>occupied_housing_units</th>\n",
       "      <th>median_home_value</th>\n",
       "      <th>median_household_income</th>\n",
       "      <th>pct_employed</th>\n",
       "      <th>pct_occupied</th>\n",
       "      <th>pct_under30</th>\n",
       "      <th>pct_over65</th>\n",
       "      <th>pct_with_kids</th>\n",
       "      <th>pct_white</th>\n",
       "      <th>pct_rich</th>\n",
       "      <th>housing_unit_density</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>02135</th>\n",
       "      <td>42780</td>\n",
       "      <td>16236.0</td>\n",
       "      <td>2.63</td>\n",
       "      <td>0.22</td>\n",
       "      <td>20834</td>\n",
       "      <td>19905</td>\n",
       "      <td>350500</td>\n",
       "      <td>48687</td>\n",
       "      <td>0.699380</td>\n",
       "      <td>0.955409</td>\n",
       "      <td>0.565932</td>\n",
       "      <td>0.087525</td>\n",
       "      <td>0.110173</td>\n",
       "      <td>0.743735</td>\n",
       "      <td>0.018857</td>\n",
       "      <td>7921.673004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02215</th>\n",
       "      <td>26125</td>\n",
       "      <td>34190.0</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.11</td>\n",
       "      <td>9324</td>\n",
       "      <td>8788</td>\n",
       "      <td>366700</td>\n",
       "      <td>35659</td>\n",
       "      <td>0.758390</td>\n",
       "      <td>0.942514</td>\n",
       "      <td>0.770911</td>\n",
       "      <td>0.031389</td>\n",
       "      <td>0.034479</td>\n",
       "      <td>0.700402</td>\n",
       "      <td>0.037032</td>\n",
       "      <td>12268.421053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02128</th>\n",
       "      <td>40508</td>\n",
       "      <td>8352.0</td>\n",
       "      <td>4.85</td>\n",
       "      <td>1.14</td>\n",
       "      <td>15854</td>\n",
       "      <td>14651</td>\n",
       "      <td>290200</td>\n",
       "      <td>48704</td>\n",
       "      <td>0.737900</td>\n",
       "      <td>0.924120</td>\n",
       "      <td>0.458666</td>\n",
       "      <td>0.067561</td>\n",
       "      <td>0.348986</td>\n",
       "      <td>0.647847</td>\n",
       "      <td>0.019369</td>\n",
       "      <td>3268.865979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02113</th>\n",
       "      <td>6915</td>\n",
       "      <td>68665.0</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4682</td>\n",
       "      <td>4336</td>\n",
       "      <td>449100</td>\n",
       "      <td>66636</td>\n",
       "      <td>0.855449</td>\n",
       "      <td>0.926100</td>\n",
       "      <td>0.519390</td>\n",
       "      <td>0.094723</td>\n",
       "      <td>0.037362</td>\n",
       "      <td>0.945915</td>\n",
       "      <td>0.073069</td>\n",
       "      <td>46820.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02122</th>\n",
       "      <td>23479</td>\n",
       "      <td>11554.0</td>\n",
       "      <td>2.03</td>\n",
       "      <td>0.63</td>\n",
       "      <td>8887</td>\n",
       "      <td>8066</td>\n",
       "      <td>320900</td>\n",
       "      <td>50192</td>\n",
       "      <td>0.658695</td>\n",
       "      <td>0.907618</td>\n",
       "      <td>0.439284</td>\n",
       "      <td>0.081441</td>\n",
       "      <td>0.377882</td>\n",
       "      <td>0.373355</td>\n",
       "      <td>0.017333</td>\n",
       "      <td>4377.832512</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       population  population_density  land_area_in_sqmi  water_area_in_sqmi  \\\n",
       "02135       42780             16236.0               2.63                0.22   \n",
       "02215       26125             34190.0               0.76                0.11   \n",
       "02128       40508              8352.0               4.85                1.14   \n",
       "02113        6915             68665.0               0.10                0.00   \n",
       "02122       23479             11554.0               2.03                0.63   \n",
       "\n",
       "       housing_units  occupied_housing_units  median_home_value  \\\n",
       "02135          20834                   19905             350500   \n",
       "02215           9324                    8788             366700   \n",
       "02128          15854                   14651             290200   \n",
       "02113           4682                    4336             449100   \n",
       "02122           8887                    8066             320900   \n",
       "\n",
       "       median_household_income  pct_employed  pct_occupied  pct_under30  \\\n",
       "02135                    48687      0.699380      0.955409     0.565932   \n",
       "02215                    35659      0.758390      0.942514     0.770911   \n",
       "02128                    48704      0.737900      0.924120     0.458666   \n",
       "02113                    66636      0.855449      0.926100     0.519390   \n",
       "02122                    50192      0.658695      0.907618     0.439284   \n",
       "\n",
       "       pct_over65  pct_with_kids  pct_white  pct_rich  housing_unit_density  \n",
       "02135    0.087525       0.110173   0.743735  0.018857           7921.673004  \n",
       "02215    0.031389       0.034479   0.700402  0.037032          12268.421053  \n",
       "02128    0.067561       0.348986   0.647847  0.019369           3268.865979  \n",
       "02113    0.094723       0.037362   0.945915  0.073069          46820.000000  \n",
       "02122    0.081441       0.377882   0.373355  0.017333           4377.832512  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# extract unique zips and look up\n",
    "unqZips = condos['ZIPCODE'].unique()\n",
    "# display(unqZips)\n",
    "zipData = {}\n",
    "for zc in unqZips:\n",
    "    zd = search.by_zipcode(zc)   \n",
    "    # too many fields, with nested sub-fields...    \n",
    "    zd = zd.to_dict()\n",
    "    # filter to the columns that we want    \n",
    "    zf = {}\n",
    "    # copy keys that don't need modification over\n",
    "    for zk in zipCols:\n",
    "        zf[zk] = zd[zk]    \n",
    "    # manually extract earnings status\n",
    "    es = zd['employment_status'][0]['values']\n",
    "    d = pd.DataFrame.from_records(es)    \n",
    "    d.set_index('x', inplace=True)    \n",
    "    # TODO: clean up subscripting/parens here\n",
    "    zf['pct_employed'] = ((d.loc['Worked Full-time With Earnings'] + d.loc['Worked Part-time With Earnings']) / d['y'].sum()).values[0]\n",
    "    # calculate percentage of owner-occupied\n",
    "    zf['pct_occupied'] = (zf['occupied_housing_units'] / zf['housing_units'])\n",
    "    # manually extract population below 30 and above 65\n",
    "    pa = zd['population_by_age'][0]['values']\n",
    "    d = pd.DataFrame.from_records(pa)    \n",
    "    d.set_index('x', inplace=True) \n",
    "    zf['pct_under30'] = (d.loc[:'25-29'].sum() / d['y'].sum()).values[0]\n",
    "    zf['pct_over65'] = (d.loc['65-69':].sum() / d['y'].sum()).values[0]\n",
    "    # manually extract population with kids\n",
    "    pk = zd['households_with_kids'][0]['values']\n",
    "    d = pd.DataFrame.from_records(pk)    \n",
    "    d.set_index('x', inplace=True)    \n",
    "    zf['pct_with_kids'] = d.loc['Households With Kids'].sum() / d['y'].sum()\n",
    "    # extract percent white\n",
    "    p = zd['population_by_race'][0]['values']\n",
    "    d = pd.DataFrame.from_records(p)    \n",
    "    d.set_index('x', inplace=True)    \n",
    "    zf['pct_white'] = d['y'].loc['White'] / d['y'].sum()\n",
    "    # extract percent rich (hhi > 200k)\n",
    "    p = zd['household_income'][0]['values']\n",
    "    d = pd.DataFrame.from_records(p)    \n",
    "    d.set_index('x', inplace=True)    \n",
    "    zf['pct_rich'] = d['y'].loc['$200,000+'] / d['y'].sum()\n",
    "    # save filtered/extracted dictionary\n",
    "    zipData[zc] = zf\n",
    "\n",
    "# build zip code dataframe\n",
    "zdf = pd.DataFrame.from_dict(zipData, orient='index')\n",
    "# calculate housing unit density\n",
    "zdf['housing_unit_density'] = zdf['housing_units'] / zdf['land_area_in_sqmi']\n",
    "print('example zip code statistics to be merged with property data:')\n",
    "display(zdf.sample(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PID</th>\n",
       "      <th>CM_ID</th>\n",
       "      <th>ZIPCODE</th>\n",
       "      <th>LU</th>\n",
       "      <th>OWN_OCC</th>\n",
       "      <th>OWNER</th>\n",
       "      <th>MAIL_ZIPCODE</th>\n",
       "      <th>AV_LAND</th>\n",
       "      <th>AV_BLDG</th>\n",
       "      <th>AV_TOTAL</th>\n",
       "      <th>GROSS_TAX</th>\n",
       "      <th>LAND_SF</th>\n",
       "      <th>YR_BUILT</th>\n",
       "      <th>YR_REMOD</th>\n",
       "      <th>LIVING_AREA</th>\n",
       "      <th>NUM_FLOORS</th>\n",
       "      <th>S_NUM_BLDG</th>\n",
       "      <th>S_BLDG_STYL</th>\n",
       "      <th>S_UNIT_RES</th>\n",
       "      <th>S_UNIT_COM</th>\n",
       "      <th>S_UNIT_RC</th>\n",
       "      <th>S_EXT_FIN</th>\n",
       "      <th>S_EXT_CND</th>\n",
       "      <th>U_BASE_FLOOR</th>\n",
       "      <th>U_NUM_PARK</th>\n",
       "      <th>U_CORNER</th>\n",
       "      <th>U_ORIENT</th>\n",
       "      <th>U_TOT_RMS</th>\n",
       "      <th>U_BDRMS</th>\n",
       "      <th>U_FULL_BTH</th>\n",
       "      <th>U_HALF_BTH</th>\n",
       "      <th>U_BTH_STYLE</th>\n",
       "      <th>U_BTH_STYLE2</th>\n",
       "      <th>U_BTH_STYLE3</th>\n",
       "      <th>U_KITCH_TYPE</th>\n",
       "      <th>U_KITCH_STYLE</th>\n",
       "      <th>U_HEAT_TYP</th>\n",
       "      <th>U_AC</th>\n",
       "      <th>U_FPLACE</th>\n",
       "      <th>U_INT_FIN</th>\n",
       "      <th>U_INT_CND</th>\n",
       "      <th>U_VIEW</th>\n",
       "      <th>n_lux</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>is_in_complex</th>\n",
       "      <th>age</th>\n",
       "      <th>population</th>\n",
       "      <th>population_density</th>\n",
       "      <th>land_area_in_sqmi</th>\n",
       "      <th>water_area_in_sqmi</th>\n",
       "      <th>housing_units</th>\n",
       "      <th>occupied_housing_units</th>\n",
       "      <th>median_home_value</th>\n",
       "      <th>median_household_income</th>\n",
       "      <th>pct_employed</th>\n",
       "      <th>pct_occupied</th>\n",
       "      <th>pct_under30</th>\n",
       "      <th>pct_over65</th>\n",
       "      <th>pct_with_kids</th>\n",
       "      <th>pct_white</th>\n",
       "      <th>pct_rich</th>\n",
       "      <th>housing_unit_density</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>306973018</th>\n",
       "      <td>306973018</td>\n",
       "      <td>3.069730e+08</td>\n",
       "      <td>02118</td>\n",
       "      <td>CD</td>\n",
       "      <td>Y</td>\n",
       "      <td>LIPTON JOHN</td>\n",
       "      <td>2118</td>\n",
       "      <td>0</td>\n",
       "      <td>724800</td>\n",
       "      <td>724800</td>\n",
       "      <td>763939</td>\n",
       "      <td>1004.0</td>\n",
       "      <td>1999.0</td>\n",
       "      <td>1999.0</td>\n",
       "      <td>1004.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>MR</td>\n",
       "      <td>28.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B</td>\n",
       "      <td>G</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>S</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>S</td>\n",
       "      <td>F</td>\n",
       "      <td>C</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>BOSTON SOUTH END</td>\n",
       "      <td>False</td>\n",
       "      <td>20.0</td>\n",
       "      <td>26498</td>\n",
       "      <td>24075.0</td>\n",
       "      <td>1.10</td>\n",
       "      <td>0.01</td>\n",
       "      <td>12967</td>\n",
       "      <td>12314</td>\n",
       "      <td>561100</td>\n",
       "      <td>54152</td>\n",
       "      <td>0.711943</td>\n",
       "      <td>0.949641</td>\n",
       "      <td>0.387396</td>\n",
       "      <td>0.079758</td>\n",
       "      <td>0.171350</td>\n",
       "      <td>0.538003</td>\n",
       "      <td>0.126882</td>\n",
       "      <td>11788.181818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>602655062</th>\n",
       "      <td>602655062</td>\n",
       "      <td>6.026550e+08</td>\n",
       "      <td>02210</td>\n",
       "      <td>CD</td>\n",
       "      <td>N</td>\n",
       "      <td>MAHANAMA THUSITH</td>\n",
       "      <td>2110</td>\n",
       "      <td>0</td>\n",
       "      <td>668600</td>\n",
       "      <td>668600</td>\n",
       "      <td>704704</td>\n",
       "      <td>747.0</td>\n",
       "      <td>1899.0</td>\n",
       "      <td>2007.0</td>\n",
       "      <td>747.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>MR</td>\n",
       "      <td>89.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B</td>\n",
       "      <td>G</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>F</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>M</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>M</td>\n",
       "      <td>F</td>\n",
       "      <td>C</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>E</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>BOSTON</td>\n",
       "      <td>False</td>\n",
       "      <td>12.0</td>\n",
       "      <td>2090</td>\n",
       "      <td>2256.0</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.54</td>\n",
       "      <td>1358</td>\n",
       "      <td>1140</td>\n",
       "      <td>500000</td>\n",
       "      <td>110333</td>\n",
       "      <td>0.911210</td>\n",
       "      <td>0.839470</td>\n",
       "      <td>0.383516</td>\n",
       "      <td>0.040370</td>\n",
       "      <td>0.059649</td>\n",
       "      <td>0.881818</td>\n",
       "      <td>0.271760</td>\n",
       "      <td>1460.215054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203090002</th>\n",
       "      <td>203090002</td>\n",
       "      <td>2.030900e+08</td>\n",
       "      <td>02129</td>\n",
       "      <td>CD</td>\n",
       "      <td>N</td>\n",
       "      <td>ARONIS CHRISTOPHER S</td>\n",
       "      <td>2129</td>\n",
       "      <td>0</td>\n",
       "      <td>546500</td>\n",
       "      <td>546500</td>\n",
       "      <td>576011</td>\n",
       "      <td>1084.0</td>\n",
       "      <td>1880.0</td>\n",
       "      <td>1985.0</td>\n",
       "      <td>1084.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>RM</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>A</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>B</td>\n",
       "      <td>5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>S</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>S</td>\n",
       "      <td>E</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>E</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>CHARLESTOWN</td>\n",
       "      <td>False</td>\n",
       "      <td>34.0</td>\n",
       "      <td>16439</td>\n",
       "      <td>12192.0</td>\n",
       "      <td>1.35</td>\n",
       "      <td>0.49</td>\n",
       "      <td>8648</td>\n",
       "      <td>8033</td>\n",
       "      <td>489900</td>\n",
       "      <td>86346</td>\n",
       "      <td>0.752229</td>\n",
       "      <td>0.928885</td>\n",
       "      <td>0.381605</td>\n",
       "      <td>0.098037</td>\n",
       "      <td>0.197062</td>\n",
       "      <td>0.800779</td>\n",
       "      <td>0.172699</td>\n",
       "      <td>6405.925926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2203713026</th>\n",
       "      <td>2203713026</td>\n",
       "      <td>2.203713e+09</td>\n",
       "      <td>02135</td>\n",
       "      <td>CD</td>\n",
       "      <td>N</td>\n",
       "      <td>MAGNANI ALAINA M</td>\n",
       "      <td>2135</td>\n",
       "      <td>0</td>\n",
       "      <td>217700</td>\n",
       "      <td>217700</td>\n",
       "      <td>229456</td>\n",
       "      <td>491.0</td>\n",
       "      <td>1964.0</td>\n",
       "      <td>1964.0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>MR</td>\n",
       "      <td>53.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V</td>\n",
       "      <td>A</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>F</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>S</td>\n",
       "      <td>W</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>BRIGHTON</td>\n",
       "      <td>False</td>\n",
       "      <td>55.0</td>\n",
       "      <td>42780</td>\n",
       "      <td>16236.0</td>\n",
       "      <td>2.63</td>\n",
       "      <td>0.22</td>\n",
       "      <td>20834</td>\n",
       "      <td>19905</td>\n",
       "      <td>350500</td>\n",
       "      <td>48687</td>\n",
       "      <td>0.699380</td>\n",
       "      <td>0.955409</td>\n",
       "      <td>0.565932</td>\n",
       "      <td>0.087525</td>\n",
       "      <td>0.110173</td>\n",
       "      <td>0.743735</td>\n",
       "      <td>0.018857</td>\n",
       "      <td>7921.673004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1704668002</th>\n",
       "      <td>1704668002</td>\n",
       "      <td>1.704668e+09</td>\n",
       "      <td>02124</td>\n",
       "      <td>CD</td>\n",
       "      <td>Y</td>\n",
       "      <td>SHEPARD CLARE</td>\n",
       "      <td>2124</td>\n",
       "      <td>0</td>\n",
       "      <td>412400</td>\n",
       "      <td>412400</td>\n",
       "      <td>434670</td>\n",
       "      <td>1156.0</td>\n",
       "      <td>1899.0</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>1156.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>FS</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>G</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>T</td>\n",
       "      <td>6</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>M</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>DORCHESTER CENTER</td>\n",
       "      <td>False</td>\n",
       "      <td>15.0</td>\n",
       "      <td>47783</td>\n",
       "      <td>15913.0</td>\n",
       "      <td>3.00</td>\n",
       "      <td>0.07</td>\n",
       "      <td>19053</td>\n",
       "      <td>17459</td>\n",
       "      <td>306500</td>\n",
       "      <td>48841</td>\n",
       "      <td>0.676657</td>\n",
       "      <td>0.916339</td>\n",
       "      <td>0.470216</td>\n",
       "      <td>0.086052</td>\n",
       "      <td>0.382954</td>\n",
       "      <td>0.210472</td>\n",
       "      <td>0.026745</td>\n",
       "      <td>6351.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   PID         CM_ID ZIPCODE  LU OWN_OCC  \\\n",
       "PID                                                        \n",
       "306973018    306973018  3.069730e+08   02118  CD       Y   \n",
       "602655062    602655062  6.026550e+08   02210  CD       N   \n",
       "203090002    203090002  2.030900e+08   02129  CD       N   \n",
       "2203713026  2203713026  2.203713e+09   02135  CD       N   \n",
       "1704668002  1704668002  1.704668e+09   02124  CD       Y   \n",
       "\n",
       "                           OWNER MAIL_ZIPCODE  AV_LAND  AV_BLDG  AV_TOTAL  \\\n",
       "PID                                                                         \n",
       "306973018            LIPTON JOHN         2118        0   724800    724800   \n",
       "602655062       MAHANAMA THUSITH         2110        0   668600    668600   \n",
       "203090002   ARONIS CHRISTOPHER S         2129        0   546500    546500   \n",
       "2203713026      MAGNANI ALAINA M         2135        0   217700    217700   \n",
       "1704668002         SHEPARD CLARE         2124        0   412400    412400   \n",
       "\n",
       "            GROSS_TAX  LAND_SF  YR_BUILT  YR_REMOD  LIVING_AREA  NUM_FLOORS  \\\n",
       "PID                                                                           \n",
       "306973018      763939   1004.0    1999.0    1999.0       1004.0         1.0   \n",
       "602655062      704704    747.0    1899.0    2007.0        747.0         1.0   \n",
       "203090002      576011   1084.0    1880.0    1985.0       1084.0         2.0   \n",
       "2203713026     229456    491.0    1964.0    1964.0        491.0         1.0   \n",
       "1704668002     434670   1156.0    1899.0    2004.0       1156.0         1.0   \n",
       "\n",
       "            S_NUM_BLDG S_BLDG_STYL  S_UNIT_RES  S_UNIT_COM  S_UNIT_RC  \\\n",
       "PID                                                                     \n",
       "306973018          1.0          MR        28.0         NaN        NaN   \n",
       "602655062          1.0          MR        89.0         NaN        NaN   \n",
       "203090002          1.0          RM         3.0         NaN        NaN   \n",
       "2203713026         1.0          MR        53.0         NaN        NaN   \n",
       "1704668002         1.0          FS         6.0         NaN        NaN   \n",
       "\n",
       "           S_EXT_FIN S_EXT_CND  U_BASE_FLOOR  U_NUM_PARK U_CORNER U_ORIENT  \\\n",
       "PID                                                                          \n",
       "306973018          B         G           2.0         0.0        N        A   \n",
       "602655062          B         G           2.0         0.0        N        F   \n",
       "203090002          F         A           0.0         0.0        N        B   \n",
       "2203713026         V         A           1.0         0.0        N        F   \n",
       "1704668002         F         G           1.0         1.0        N        T   \n",
       "\n",
       "           U_TOT_RMS  U_BDRMS  U_FULL_BTH  U_HALF_BTH U_BTH_STYLE  \\\n",
       "PID                                                                 \n",
       "306973018          4      2.0         2.0         0.0           S   \n",
       "602655062          4      1.0         1.0         1.0           M   \n",
       "203090002          5      2.0         1.0         1.0           S   \n",
       "2203713026         2      0.0         1.0         0.0           S   \n",
       "1704668002         6      3.0         1.0         0.0           M   \n",
       "\n",
       "           U_BTH_STYLE2 U_BTH_STYLE3 U_KITCH_TYPE U_KITCH_STYLE U_HEAT_TYP  \\\n",
       "PID                                                                          \n",
       "306973018             S          NaN            O             S          F   \n",
       "602655062             M          NaN            F             M          F   \n",
       "203090002             M          NaN            F             S          E   \n",
       "2203713026          NaN          NaN            O             S          W   \n",
       "1704668002          NaN          NaN            O             M          F   \n",
       "\n",
       "           U_AC  U_FPLACE U_INT_FIN U_INT_CND U_VIEW  n_lux  \\\n",
       "PID                                                           \n",
       "306973018     C       0.0         N         G      A      0   \n",
       "602655062     C       0.0         N         E      A      0   \n",
       "203090002     N       0.0         E         A      A      0   \n",
       "2203713026    N       0.0         N         A      A      0   \n",
       "1704668002    N       1.0         N         G      A      0   \n",
       "\n",
       "                 Neighborhood  is_in_complex   age  population  \\\n",
       "PID                                                              \n",
       "306973018    BOSTON SOUTH END          False  20.0       26498   \n",
       "602655062              BOSTON          False  12.0        2090   \n",
       "203090002         CHARLESTOWN          False  34.0       16439   \n",
       "2203713026           BRIGHTON          False  55.0       42780   \n",
       "1704668002  DORCHESTER CENTER          False  15.0       47783   \n",
       "\n",
       "            population_density  land_area_in_sqmi  water_area_in_sqmi  \\\n",
       "PID                                                                     \n",
       "306973018              24075.0               1.10                0.01   \n",
       "602655062               2256.0               0.93                0.54   \n",
       "203090002              12192.0               1.35                0.49   \n",
       "2203713026             16236.0               2.63                0.22   \n",
       "1704668002             15913.0               3.00                0.07   \n",
       "\n",
       "            housing_units  occupied_housing_units  median_home_value  \\\n",
       "PID                                                                    \n",
       "306973018           12967                   12314             561100   \n",
       "602655062            1358                    1140             500000   \n",
       "203090002            8648                    8033             489900   \n",
       "2203713026          20834                   19905             350500   \n",
       "1704668002          19053                   17459             306500   \n",
       "\n",
       "            median_household_income  pct_employed  pct_occupied  pct_under30  \\\n",
       "PID                                                                            \n",
       "306973018                     54152      0.711943      0.949641     0.387396   \n",
       "602655062                    110333      0.911210      0.839470     0.383516   \n",
       "203090002                     86346      0.752229      0.928885     0.381605   \n",
       "2203713026                    48687      0.699380      0.955409     0.565932   \n",
       "1704668002                    48841      0.676657      0.916339     0.470216   \n",
       "\n",
       "            pct_over65  pct_with_kids  pct_white  pct_rich  \\\n",
       "PID                                                          \n",
       "306973018     0.079758       0.171350   0.538003  0.126882   \n",
       "602655062     0.040370       0.059649   0.881818  0.271760   \n",
       "203090002     0.098037       0.197062   0.800779  0.172699   \n",
       "2203713026    0.087525       0.110173   0.743735  0.018857   \n",
       "1704668002    0.086052       0.382954   0.210472  0.026745   \n",
       "\n",
       "            housing_unit_density  \n",
       "PID                               \n",
       "306973018           11788.181818  \n",
       "602655062            1460.215054  \n",
       "203090002            6405.925926  \n",
       "2203713026           7921.673004  \n",
       "1704668002           6351.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# merge zipcode data with condo data\n",
    "condos = pd.merge(condos, zdf, left_on='ZIPCODE', right_index=True, how='left')\n",
    "display(condos.sample(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select Variables for Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y variable in regressions\n",
    "targetCol = 'AV_TOTAL'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Continuous Variables\n",
    "NaN values in any of these variables will cause a given sample to be dropped."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define continuous non-categorical columns to use in regressions\n",
    "contReg = ['LIVING_AREA', 'LAND_SF', 'U_BASE_FLOOR', 'U_BDRMS',\n",
    "           'age', 'n_lux', 'U_NUM_PARK', 'U_FULL_BTH',\n",
    "           'U_HALF_BTH', 'U_FPLACE', 'S_UNIT_RES',\n",
    "           'population_density', 'water_area_in_sqmi',\n",
    "           'housing_unit_density', 'median_home_value',\n",
    "           'median_household_income', 'pct_employed', 'pct_occupied',\n",
    "           'pct_under30', 'pct_over65', 'pct_with_kids', 'pct_white', 'pct_rich']\n",
    "for ctc in contReg:\n",
    "    if ctc not in condos.columns:\n",
    "        print(ctc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Binary Variables\n",
    "OHE and get_dummies can't deal with boolean values, so might as well deal with all binary variables here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "binReg = ['OWN_OCC', 'is_in_complex', 'U_CORNER']\n",
    "# convert y/n to bit\n",
    "for bc in binReg:\n",
    "    if condos[bc].dtype == 'bool':        \n",
    "        condos[bc] = condos[bc].astype(int)\n",
    "    else:        \n",
    "        try:\n",
    "            condos[bc] = condos[bc].str.upper().map({'Y': 1, 'N': 0})\n",
    "        except:\n",
    "            pass # already int, because you re-ran this cell probably"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OWN_OCC          False\n",
       "is_in_complex    False\n",
       "U_CORNER         False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# none of these columns can have missing values that would cause sample dropping\n",
    "condos[binReg].isnull().any()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtered list of categorical variables to use in regression analysis\n",
    "catReg = ['S_BLDG_STYL', 'S_EXT_FIN', 'S_EXT_CND', 'U_ORIENT', 'U_BTH_STYLE', 'U_KITCH_TYPE',\n",
    "       'U_KITCH_STYLE', 'U_HEAT_TYP', 'U_AC', 'U_INT_FIN', 'U_INT_CND', 'U_VIEW', 'Neighborhood']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "S_BLDG_STYL      8\n",
       "S_EXT_FIN        8\n",
       "S_EXT_CND        8\n",
       "U_ORIENT         0\n",
       "U_BTH_STYLE      0\n",
       "U_KITCH_TYPE     0\n",
       "U_KITCH_STYLE    0\n",
       "U_HEAT_TYP       0\n",
       "U_AC             0\n",
       "U_INT_FIN        0\n",
       "U_INT_CND        0\n",
       "U_VIEW           0\n",
       "Neighborhood     0\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# how many rows have null values for each of these fields?\n",
    "display(condos[catReg].isnull().sum())\n",
    "# there are only 8 values that are null for 3 of the categorical columns,\n",
    "#    so it's safe to drop any rows with na in any of these columns.\n",
    "condos = condos.dropna(subset=catReg, how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>S_BLDG_STYL_CV</th>\n",
       "      <th>S_BLDG_STYL_DK</th>\n",
       "      <th>S_BLDG_STYL_DX</th>\n",
       "      <th>S_BLDG_STYL_FS</th>\n",
       "      <th>S_BLDG_STYL_HR</th>\n",
       "      <th>S_BLDG_STYL_LR</th>\n",
       "      <th>S_BLDG_STYL_MR</th>\n",
       "      <th>S_BLDG_STYL_RE</th>\n",
       "      <th>S_BLDG_STYL_RM</th>\n",
       "      <th>S_BLDG_STYL_SD</th>\n",
       "      <th>S_BLDG_STYL_TH</th>\n",
       "      <th>S_EXT_FIN_B</th>\n",
       "      <th>S_EXT_FIN_C</th>\n",
       "      <th>S_EXT_FIN_F</th>\n",
       "      <th>S_EXT_FIN_G</th>\n",
       "      <th>S_EXT_FIN_K</th>\n",
       "      <th>S_EXT_FIN_M</th>\n",
       "      <th>S_EXT_FIN_O</th>\n",
       "      <th>S_EXT_FIN_P</th>\n",
       "      <th>S_EXT_FIN_S</th>\n",
       "      <th>S_EXT_FIN_U</th>\n",
       "      <th>S_EXT_FIN_V</th>\n",
       "      <th>S_EXT_FIN_W</th>\n",
       "      <th>S_EXT_CND_E</th>\n",
       "      <th>S_EXT_CND_F</th>\n",
       "      <th>S_EXT_CND_G</th>\n",
       "      <th>S_EXT_CND_P</th>\n",
       "      <th>U_ORIENT_B</th>\n",
       "      <th>U_ORIENT_C</th>\n",
       "      <th>U_ORIENT_E</th>\n",
       "      <th>U_ORIENT_F</th>\n",
       "      <th>U_ORIENT_M</th>\n",
       "      <th>U_ORIENT_T</th>\n",
       "      <th>U_BTH_STYLE_M</th>\n",
       "      <th>U_BTH_STYLE_N</th>\n",
       "      <th>U_BTH_STYLE_S</th>\n",
       "      <th>U_KITCH_TYPE_N</th>\n",
       "      <th>U_KITCH_TYPE_O</th>\n",
       "      <th>U_KITCH_TYPE_P</th>\n",
       "      <th>U_KITCH_STYLE_M</th>\n",
       "      <th>U_KITCH_STYLE_N</th>\n",
       "      <th>U_KITCH_STYLE_S</th>\n",
       "      <th>U_HEAT_TYP_F</th>\n",
       "      <th>U_HEAT_TYP_N</th>\n",
       "      <th>U_HEAT_TYP_O</th>\n",
       "      <th>U_HEAT_TYP_P</th>\n",
       "      <th>U_HEAT_TYP_S</th>\n",
       "      <th>U_HEAT_TYP_W</th>\n",
       "      <th>U_AC_D</th>\n",
       "      <th>U_AC_N</th>\n",
       "      <th>U_INT_FIN_N</th>\n",
       "      <th>U_INT_FIN_S</th>\n",
       "      <th>U_INT_CND_E</th>\n",
       "      <th>U_INT_CND_F</th>\n",
       "      <th>U_INT_CND_G</th>\n",
       "      <th>U_INT_CND_P</th>\n",
       "      <th>U_VIEW_E</th>\n",
       "      <th>U_VIEW_F</th>\n",
       "      <th>U_VIEW_G</th>\n",
       "      <th>U_VIEW_P</th>\n",
       "      <th>U_VIEW_S</th>\n",
       "      <th>Neighborhood_BOSTON</th>\n",
       "      <th>Neighborhood_BOSTON BACK BAY</th>\n",
       "      <th>Neighborhood_BOSTON BEACON  HILL</th>\n",
       "      <th>Neighborhood_BOSTON CHINATOWN</th>\n",
       "      <th>Neighborhood_BOSTON DOWNTOWN CROSSING</th>\n",
       "      <th>Neighborhood_BOSTON FENWAY</th>\n",
       "      <th>Neighborhood_BOSTON KENMORE</th>\n",
       "      <th>Neighborhood_BOSTON NORTH END</th>\n",
       "      <th>Neighborhood_BOSTON SOUTH END</th>\n",
       "      <th>Neighborhood_BOSTON WEST END</th>\n",
       "      <th>Neighborhood_BRIGHTON</th>\n",
       "      <th>Neighborhood_BROOKLINE</th>\n",
       "      <th>Neighborhood_CHARLESTOWN</th>\n",
       "      <th>Neighborhood_DORCHESTER</th>\n",
       "      <th>Neighborhood_DORCHESTER CENTER</th>\n",
       "      <th>Neighborhood_DORCHESTER GROVE HALL</th>\n",
       "      <th>Neighborhood_DORCHESTER UPHAMS CORNER</th>\n",
       "      <th>Neighborhood_EAST BOSTON</th>\n",
       "      <th>Neighborhood_HYDE PARK</th>\n",
       "      <th>Neighborhood_JAMAICA PLAIN</th>\n",
       "      <th>Neighborhood_MATTAPAN</th>\n",
       "      <th>Neighborhood_ROSLINDALE</th>\n",
       "      <th>Neighborhood_ROXBURY</th>\n",
       "      <th>Neighborhood_ROXBURY MISSION HILL</th>\n",
       "      <th>Neighborhood_SOUTH BOSTON</th>\n",
       "      <th>Neighborhood_WEST ROXBURY</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>502550008</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502550010</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502550012</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502550014</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502551002</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           S_BLDG_STYL_CV  S_BLDG_STYL_DK  S_BLDG_STYL_DX  S_BLDG_STYL_FS  \\\n",
       "PID                                                                         \n",
       "502550008               0               0               0               0   \n",
       "502550010               0               0               0               0   \n",
       "502550012               0               0               0               0   \n",
       "502550014               0               0               0               0   \n",
       "502551002               0               0               0               0   \n",
       "\n",
       "           S_BLDG_STYL_HR  S_BLDG_STYL_LR  S_BLDG_STYL_MR  S_BLDG_STYL_RE  \\\n",
       "PID                                                                         \n",
       "502550008               0               0               0               0   \n",
       "502550010               0               0               0               0   \n",
       "502550012               0               0               0               0   \n",
       "502550014               0               0               0               0   \n",
       "502551002               0               0               0               0   \n",
       "\n",
       "           S_BLDG_STYL_RM  S_BLDG_STYL_SD  S_BLDG_STYL_TH  S_EXT_FIN_B  \\\n",
       "PID                                                                      \n",
       "502550008               1               0               0            1   \n",
       "502550010               1               0               0            1   \n",
       "502550012               1               0               0            1   \n",
       "502550014               1               0               0            1   \n",
       "502551002               1               0               0            1   \n",
       "\n",
       "           S_EXT_FIN_C  S_EXT_FIN_F  S_EXT_FIN_G  S_EXT_FIN_K  S_EXT_FIN_M  \\\n",
       "PID                                                                          \n",
       "502550008            0            0            0            0            0   \n",
       "502550010            0            0            0            0            0   \n",
       "502550012            0            0            0            0            0   \n",
       "502550014            0            0            0            0            0   \n",
       "502551002            0            0            0            0            0   \n",
       "\n",
       "           S_EXT_FIN_O  S_EXT_FIN_P  S_EXT_FIN_S  S_EXT_FIN_U  S_EXT_FIN_V  \\\n",
       "PID                                                                          \n",
       "502550008            0            0            0            0            0   \n",
       "502550010            0            0            0            0            0   \n",
       "502550012            0            0            0            0            0   \n",
       "502550014            0            0            0            0            0   \n",
       "502551002            0            0            0            0            0   \n",
       "\n",
       "           S_EXT_FIN_W  S_EXT_CND_E  S_EXT_CND_F  S_EXT_CND_G  S_EXT_CND_P  \\\n",
       "PID                                                                          \n",
       "502550008            0            0            0            1            0   \n",
       "502550010            0            0            0            1            0   \n",
       "502550012            0            0            0            1            0   \n",
       "502550014            0            0            0            1            0   \n",
       "502551002            0            0            0            1            0   \n",
       "\n",
       "           U_ORIENT_B  U_ORIENT_C  U_ORIENT_E  U_ORIENT_F  U_ORIENT_M  \\\n",
       "PID                                                                     \n",
       "502550008           0           0           0           1           0   \n",
       "502550010           0           0           0           0           0   \n",
       "502550012           0           0           0           1           0   \n",
       "502550014           0           0           0           0           0   \n",
       "502551002           1           0           0           0           0   \n",
       "\n",
       "           U_ORIENT_T  U_BTH_STYLE_M  U_BTH_STYLE_N  U_BTH_STYLE_S  \\\n",
       "PID                                                                  \n",
       "502550008           0              1              0              0   \n",
       "502550010           0              0              0              1   \n",
       "502550012           0              1              0              0   \n",
       "502550014           0              0              1              0   \n",
       "502551002           0              0              0              1   \n",
       "\n",
       "           U_KITCH_TYPE_N  U_KITCH_TYPE_O  U_KITCH_TYPE_P  U_KITCH_STYLE_M  \\\n",
       "PID                                                                          \n",
       "502550008               0               1               0                1   \n",
       "502550010               0               1               0                0   \n",
       "502550012               0               1               0                1   \n",
       "502550014               0               1               0                0   \n",
       "502551002               0               0               1                0   \n",
       "\n",
       "           U_KITCH_STYLE_N  U_KITCH_STYLE_S  U_HEAT_TYP_F  U_HEAT_TYP_N  \\\n",
       "PID                                                                       \n",
       "502550008                0                0             0             0   \n",
       "502550010                0                1             0             0   \n",
       "502550012                0                0             0             0   \n",
       "502550014                1                0             0             0   \n",
       "502551002                0                1             0             0   \n",
       "\n",
       "           U_HEAT_TYP_O  U_HEAT_TYP_P  U_HEAT_TYP_S  U_HEAT_TYP_W  U_AC_D  \\\n",
       "PID                                                                         \n",
       "502550008             0             0             0             1       0   \n",
       "502550010             0             0             0             1       0   \n",
       "502550012             0             0             0             1       0   \n",
       "502550014             0             0             0             1       0   \n",
       "502551002             0             0             0             1       0   \n",
       "\n",
       "           U_AC_N  U_INT_FIN_N  U_INT_FIN_S  U_INT_CND_E  U_INT_CND_F  \\\n",
       "PID                                                                     \n",
       "502550008       1            1            0            0            0   \n",
       "502550010       1            1            0            0            0   \n",
       "502550012       1            1            0            0            0   \n",
       "502550014       1            0            1            0            0   \n",
       "502551002       1            1            0            0            0   \n",
       "\n",
       "           U_INT_CND_G  U_INT_CND_P  U_VIEW_E  U_VIEW_F  U_VIEW_G  U_VIEW_P  \\\n",
       "PID                                                                           \n",
       "502550008            1            0         0         0         0         0   \n",
       "502550010            0            0         0         0         0         0   \n",
       "502550012            0            0         0         0         1         0   \n",
       "502550014            0            0         0         0         1         0   \n",
       "502551002            0            0         0         1         0         0   \n",
       "\n",
       "           U_VIEW_S  Neighborhood_BOSTON  Neighborhood_BOSTON BACK BAY  \\\n",
       "PID                                                                      \n",
       "502550008         0                    0                             0   \n",
       "502550010         0                    0                             0   \n",
       "502550012         0                    0                             0   \n",
       "502550014         0                    0                             0   \n",
       "502551002         0                    0                             0   \n",
       "\n",
       "           Neighborhood_BOSTON BEACON  HILL  Neighborhood_BOSTON CHINATOWN  \\\n",
       "PID                                                                          \n",
       "502550008                                 1                              0   \n",
       "502550010                                 1                              0   \n",
       "502550012                                 1                              0   \n",
       "502550014                                 1                              0   \n",
       "502551002                                 1                              0   \n",
       "\n",
       "           Neighborhood_BOSTON DOWNTOWN CROSSING  Neighborhood_BOSTON FENWAY  \\\n",
       "PID                                                                            \n",
       "502550008                                      0                           0   \n",
       "502550010                                      0                           0   \n",
       "502550012                                      0                           0   \n",
       "502550014                                      0                           0   \n",
       "502551002                                      0                           0   \n",
       "\n",
       "           Neighborhood_BOSTON KENMORE  Neighborhood_BOSTON NORTH END  \\\n",
       "PID                                                                     \n",
       "502550008                            0                              0   \n",
       "502550010                            0                              0   \n",
       "502550012                            0                              0   \n",
       "502550014                            0                              0   \n",
       "502551002                            0                              0   \n",
       "\n",
       "           Neighborhood_BOSTON SOUTH END  Neighborhood_BOSTON WEST END  \\\n",
       "PID                                                                      \n",
       "502550008                              0                             0   \n",
       "502550010                              0                             0   \n",
       "502550012                              0                             0   \n",
       "502550014                              0                             0   \n",
       "502551002                              0                             0   \n",
       "\n",
       "           Neighborhood_BRIGHTON  Neighborhood_BROOKLINE  \\\n",
       "PID                                                        \n",
       "502550008                      0                       0   \n",
       "502550010                      0                       0   \n",
       "502550012                      0                       0   \n",
       "502550014                      0                       0   \n",
       "502551002                      0                       0   \n",
       "\n",
       "           Neighborhood_CHARLESTOWN  Neighborhood_DORCHESTER  \\\n",
       "PID                                                            \n",
       "502550008                         0                        0   \n",
       "502550010                         0                        0   \n",
       "502550012                         0                        0   \n",
       "502550014                         0                        0   \n",
       "502551002                         0                        0   \n",
       "\n",
       "           Neighborhood_DORCHESTER CENTER  Neighborhood_DORCHESTER GROVE HALL  \\\n",
       "PID                                                                             \n",
       "502550008                               0                                   0   \n",
       "502550010                               0                                   0   \n",
       "502550012                               0                                   0   \n",
       "502550014                               0                                   0   \n",
       "502551002                               0                                   0   \n",
       "\n",
       "           Neighborhood_DORCHESTER UPHAMS CORNER  Neighborhood_EAST BOSTON  \\\n",
       "PID                                                                          \n",
       "502550008                                      0                         0   \n",
       "502550010                                      0                         0   \n",
       "502550012                                      0                         0   \n",
       "502550014                                      0                         0   \n",
       "502551002                                      0                         0   \n",
       "\n",
       "           Neighborhood_HYDE PARK  Neighborhood_JAMAICA PLAIN  \\\n",
       "PID                                                             \n",
       "502550008                       0                           0   \n",
       "502550010                       0                           0   \n",
       "502550012                       0                           0   \n",
       "502550014                       0                           0   \n",
       "502551002                       0                           0   \n",
       "\n",
       "           Neighborhood_MATTAPAN  Neighborhood_ROSLINDALE  \\\n",
       "PID                                                         \n",
       "502550008                      0                        0   \n",
       "502550010                      0                        0   \n",
       "502550012                      0                        0   \n",
       "502550014                      0                        0   \n",
       "502551002                      0                        0   \n",
       "\n",
       "           Neighborhood_ROXBURY  Neighborhood_ROXBURY MISSION HILL  \\\n",
       "PID                                                                  \n",
       "502550008                     0                                  0   \n",
       "502550010                     0                                  0   \n",
       "502550012                     0                                  0   \n",
       "502550014                     0                                  0   \n",
       "502551002                     0                                  0   \n",
       "\n",
       "           Neighborhood_SOUTH BOSTON  Neighborhood_WEST ROXBURY  \n",
       "PID                                                              \n",
       "502550008                          0                          0  \n",
       "502550010                          0                          0  \n",
       "502550012                          0                          0  \n",
       "502550014                          0                          0  \n",
       "502551002                          0                          0  "
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract categorical features and convert to dummies, \n",
    "#    leaving one indicator column out to address collinearity\n",
    "# use pandas to get dummy variables for these categorical columns\n",
    "regDummies = pd.get_dummies(condos[catReg], drop_first=True)\n",
    "dummyCols = list(regDummies.columns)\n",
    "regDummies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13 categorical variables converted into 87 binary features\n"
     ]
    }
   ],
   "source": [
    "print(f'{len(catReg)} categorical variables converted into {len(regDummies.columns)} binary features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# none of the dummy values are nan by design\n",
    "regDummies.isnull().any().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "S_BLDG_STYL_CV    0.030110\n",
       "S_EXT_FIN_K       0.045726\n",
       "S_EXT_FIN_P       0.037422\n",
       "S_EXT_FIN_U       0.005744\n",
       "S_EXT_CND_F       0.040988\n",
       "S_EXT_CND_P       0.012185\n",
       "U_ORIENT_E        0.045906\n",
       "U_KITCH_TYPE_N    0.043137\n",
       "U_HEAT_TYP_N      0.005744\n",
       "U_HEAT_TYP_O      0.014069\n",
       "U_HEAT_TYP_S      0.038930\n",
       "U_INT_FIN_S       0.030919\n",
       "U_INT_CND_P       0.021488\n",
       "U_VIEW_S          0.012185\n",
       "dtype: float64"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# what's variance like among ourcategorical feature dummies?\n",
    "#    extremely low variance may signal a candidate for filtering\n",
    "regDummies.std().loc[regDummies.std() < 0.05]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Although many of the dummy variables generated from categorical features have low variance, the low variance does seem to reflect rare but significant property-related features, such as having poor exterior condition or substandard interior. Therefore, it doesn't seem like a blanket variance filter would be a good choice."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Align samples across features and target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# copy original input data so we still have it for classification/clustering later\n",
    "contData = condos[contReg].copy()\n",
    "binData = condos[binReg].copy()\n",
    "tgtData = condos[targetCol].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "regData = pd.concat([contData, binData, tgtData, regDummies.copy()], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop any rows with nan values since they must be in continuous variables\n",
    "regData = regData.dropna(how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate out target with aligned NaN removals\n",
    "regTgt = regData[targetCol]\n",
    "regData.drop(targetCol, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pair plot of selected properties\n",
    "# sns.pairplot(contData[([targetCol] + ['age', 'n_lux',\n",
    "#               'population_density', 'median_household_income', 'pct_employed', 'pct_occupied',\n",
    "#               'pct_under30', 'pct_with_kids', 'pct_white', 'pct_rich'])])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning model imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sklearn and related imports\n",
    "# pre-processing\n",
    "import sklearn\n",
    "from sklearn.preprocessing import StandardScaler, PolynomialFeatures\n",
    "from sklearn.feature_selection import SelectKBest, SelectPercentile, f_regression\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.compose import make_column_transformer\n",
    "# regressions\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "# classifiers\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC    # \"Support vector classifier\"\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "# cv/metrics\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, scorer, accuracy_score\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "# ignore data conversion warnings when fitting models\n",
    "from sklearn.exceptions import DataConversionWarning\n",
    "warnings.filterwarnings(action='ignore', category=DataConversionWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression Variants (including PCA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setting for training part used\n",
    "trainPct = 0.25\n",
    "nFolds = 8 # for cross-validation\n",
    "# helper function to run and score model/pipeline\n",
    "# should be agnostic to the nature of the model itself, which could be a pipeline\n",
    "def run_model(model, df, y):\n",
    "    # return two dictionaries: table fields and diagnostic/non-scalar data\n",
    "    res = {}\n",
    "    data = {}\n",
    "    train_X, test_X, train_y, test_y = train_test_split(df, y, test_size=trainPct)\n",
    "    model.fit(train_X, train_y)\n",
    "    predictions = model.predict(test_X)\n",
    "    \n",
    "    def add_cv_stats(scorer='r2'):\n",
    "        cvr2 = cross_val_score(model, df, y, cv=nFolds, scoring=scorer)\n",
    "        # return vector of r2 as data field, but calculate table cols as well\n",
    "        data['cv_r2'] = cvr2\n",
    "        res['cv_mean_r2'] = cvr2.mean()\n",
    "        res['cv_std_r2'] = cvr2.std()  \n",
    "    \n",
    "    def add_linear_stats(lr):\n",
    "        data['coefs'] = lr.coef_\n",
    "        res['n_coeff'] = lr.coef_.size\n",
    "        data['intercept'] = lr.intercept_        \n",
    "        res['mae'] = mean_absolute_error(test_y, predictions)    \n",
    "        res['rmse'] = mean_squared_error(test_y, predictions) ** (1/2)\n",
    "        res['r2'] = r2_score(test_y, predictions)\n",
    "        \n",
    "    if hasattr(model, 'named_steps'):\n",
    "        steps = model.named_steps # alias\n",
    "    else:\n",
    "        # replicate if not pipeline for consistent logic below\n",
    "        steps = {f'{type(model).__name__}'.lower(): model}\n",
    "    \n",
    "    if 'linearregression' in steps:\n",
    "        add_cv_stats(scorer='r2')\n",
    "        add_linear_stats(steps['linearregression'])\n",
    "        \n",
    "    if 'pca' in steps:\n",
    "        pca = model.named_steps['pca'] \n",
    "        res['pca_explained_pct'] = sum(pca.explained_variance_ratio_)\n",
    "\n",
    "    data['predictions'] = predictions\n",
    "    res['score'] = model.score(test_X, test_y)\n",
    "    \n",
    "    try:\n",
    "        res['is_accuracy_score'] = accuracy_score(y, model.predict(df))\n",
    "        res['oos_accuracy_score'] = accuracy_score(test_y, predictions)\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    return res, data\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocessing depends on column type, so use a column transformer to apply scaler to only\n",
    "#    continuous columns\n",
    "def continuous_scaler(scaler):\n",
    "    return make_column_transformer((scaler, contReg), remainder='passthrough')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run with and without standardization\n",
    "# keep models in a dictionary\n",
    "linMdl = {}\n",
    "linMdl['unscaled_lr'] = LinearRegression(fit_intercept=True)\n",
    "linMdl['unscaled_noIntercept_lr'] = LinearRegression(fit_intercept=False)\n",
    "linMdl['std_lr'] = make_pipeline(StandardScaler(), LinearRegression(fit_intercept=True))\n",
    "linMdl['norm_lr'] = LinearRegression(fit_intercept=True, normalize=True)\n",
    "# apply principal components\n",
    "components_range = range(1, 11)\n",
    "for ncomp in components_range:\n",
    "    linMdl['std_pca{}_lr'.format(ncomp)] = make_pipeline(continuous_scaler(StandardScaler()), \n",
    "                                                         PCA(n_components=ncomp), \n",
    "                                                         LinearRegression(fit_intercept=True))\n",
    "# # look at polynomial regressions; square and cubic\n",
    "# poly_range = range(2, 3)\n",
    "# for pfc in poly_range: # iterate over polynomial feature count\n",
    "#     linMdl['std_poly{}_lr'.format(pfc)] = make_pipeline(make_scaler(StandardScaler()),\n",
    "#                                                         PolynomialFeatures(degree=pfc),\n",
    "#                                                         LinearRegression(fit_intercept=True))\n",
    "    \n",
    "# look at selecting best percentile features using f_regression\n",
    "# fcRng = range(2, 9)\n",
    "# percentiles, as integer, to keep\n",
    "fcRng = range(5, 50, 5)\n",
    "for fc in fcRng:\n",
    "    linMdl['std_kBest{}_lr'.format(fc)] = make_pipeline(continuous_scaler(StandardScaler()),\n",
    "                                                        SelectPercentile(f_regression,\n",
    "                                                                         percentile=fc),\n",
    "                                                        LinearRegression(fit_intercept=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# another take at linear pipelines and feature selection\n",
    "# \n",
    "\n",
    "linMdl = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'fitting std_kBest45_lr'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\sklearn\\feature_selection\\univariate_selection.py:299: RuntimeWarning: invalid value encountered in true_divide\n",
      "  corr /= X_norms\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in greater\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in less\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:1821: RuntimeWarning: invalid value encountered in less_equal\n",
      "  cond2 = cond0 & (x <= self.a)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\sklearn\\feature_selection\\univariate_selection.py:299: RuntimeWarning: invalid value encountered in true_divide\n",
      "  corr /= X_norms\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in greater\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in less\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:1821: RuntimeWarning: invalid value encountered in less_equal\n",
      "  cond2 = cond0 & (x <= self.a)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\sklearn\\feature_selection\\univariate_selection.py:299: RuntimeWarning: invalid value encountered in true_divide\n",
      "  corr /= X_norms\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in greater\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in less\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:1821: RuntimeWarning: invalid value encountered in less_equal\n",
      "  cond2 = cond0 & (x <= self.a)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\sklearn\\feature_selection\\univariate_selection.py:299: RuntimeWarning: invalid value encountered in true_divide\n",
      "  corr /= X_norms\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in greater\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in less\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:1821: RuntimeWarning: invalid value encountered in less_equal\n",
      "  cond2 = cond0 & (x <= self.a)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\sklearn\\feature_selection\\univariate_selection.py:299: RuntimeWarning: invalid value encountered in true_divide\n",
      "  corr /= X_norms\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in greater\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:879: RuntimeWarning: invalid value encountered in less\n",
      "  return (self.a < x) & (x < self.b)\n",
      "C:\\Users\\n0305827\\AppData\\Local\\Continuum\\anaconda3\\envs\\py37\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:1821: RuntimeWarning: invalid value encountered in less_equal\n",
      "  cond2 = cond0 & (x <= self.a)\n"
     ]
    }
   ],
   "source": [
    "# apply the model testing function above\n",
    "linMdlRes = {}\n",
    "for k, mdl in list(linMdl.items()):\n",
    "    clear_output(wait=True)\n",
    "    display('fitting {}'.format(k))\n",
    "    linMdlRes[k] = run_model(mdl, regData, regTgt)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cv_mean_r2</th>\n",
       "      <th>cv_std_r2</th>\n",
       "      <th>n_coeff</th>\n",
       "      <th>mae</th>\n",
       "      <th>rmse</th>\n",
       "      <th>r2</th>\n",
       "      <th>score</th>\n",
       "      <th>is_accuracy_score</th>\n",
       "      <th>oos_accuracy_score</th>\n",
       "      <th>pca_explained_pct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>norm_lr</th>\n",
       "      <td>-440368777051548579528704.000</td>\n",
       "      <td>765316432812680247508992.000</td>\n",
       "      <td>113</td>\n",
       "      <td>89450.309</td>\n",
       "      <td>129868.311</td>\n",
       "      <td>0.821</td>\n",
       "      <td>0.821</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest10_lr</th>\n",
       "      <td>0.234</td>\n",
       "      <td>0.612</td>\n",
       "      <td>12</td>\n",
       "      <td>109669.750</td>\n",
       "      <td>158603.304</td>\n",
       "      <td>0.736</td>\n",
       "      <td>0.736</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest15_lr</th>\n",
       "      <td>0.378</td>\n",
       "      <td>0.513</td>\n",
       "      <td>17</td>\n",
       "      <td>107982.595</td>\n",
       "      <td>156396.261</td>\n",
       "      <td>0.742</td>\n",
       "      <td>0.742</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest20_lr</th>\n",
       "      <td>0.491</td>\n",
       "      <td>0.258</td>\n",
       "      <td>23</td>\n",
       "      <td>107677.423</td>\n",
       "      <td>157238.313</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.750</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest25_lr</th>\n",
       "      <td>0.473</td>\n",
       "      <td>0.272</td>\n",
       "      <td>28</td>\n",
       "      <td>98323.827</td>\n",
       "      <td>142234.782</td>\n",
       "      <td>0.786</td>\n",
       "      <td>0.786</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest30_lr</th>\n",
       "      <td>0.562</td>\n",
       "      <td>0.229</td>\n",
       "      <td>34</td>\n",
       "      <td>96460.662</td>\n",
       "      <td>140511.012</td>\n",
       "      <td>0.793</td>\n",
       "      <td>0.793</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest35_lr</th>\n",
       "      <td>0.571</td>\n",
       "      <td>0.236</td>\n",
       "      <td>40</td>\n",
       "      <td>95258.178</td>\n",
       "      <td>137727.173</td>\n",
       "      <td>0.803</td>\n",
       "      <td>0.803</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest40_lr</th>\n",
       "      <td>0.577</td>\n",
       "      <td>0.225</td>\n",
       "      <td>45</td>\n",
       "      <td>94291.720</td>\n",
       "      <td>138799.275</td>\n",
       "      <td>0.794</td>\n",
       "      <td>0.794</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest45_lr</th>\n",
       "      <td>0.576</td>\n",
       "      <td>0.243</td>\n",
       "      <td>51</td>\n",
       "      <td>94890.419</td>\n",
       "      <td>137744.109</td>\n",
       "      <td>0.800</td>\n",
       "      <td>0.800</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest5_lr</th>\n",
       "      <td>0.171</td>\n",
       "      <td>0.542</td>\n",
       "      <td>6</td>\n",
       "      <td>127609.832</td>\n",
       "      <td>179204.935</td>\n",
       "      <td>0.660</td>\n",
       "      <td>0.660</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_lr</th>\n",
       "      <td>-3339445334256560111616.000</td>\n",
       "      <td>4289501379982355070976.000</td>\n",
       "      <td>113</td>\n",
       "      <td>88737.220</td>\n",
       "      <td>129438.012</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.826</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca10_lr</th>\n",
       "      <td>0.329</td>\n",
       "      <td>0.368</td>\n",
       "      <td>10</td>\n",
       "      <td>121423.807</td>\n",
       "      <td>171687.925</td>\n",
       "      <td>0.689</td>\n",
       "      <td>0.689</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca1_lr</th>\n",
       "      <td>-0.362</td>\n",
       "      <td>0.417</td>\n",
       "      <td>1</td>\n",
       "      <td>211408.806</td>\n",
       "      <td>286644.592</td>\n",
       "      <td>0.139</td>\n",
       "      <td>0.139</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca2_lr</th>\n",
       "      <td>-0.017</td>\n",
       "      <td>0.504</td>\n",
       "      <td>2</td>\n",
       "      <td>174395.700</td>\n",
       "      <td>240532.973</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.400</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca3_lr</th>\n",
       "      <td>0.320</td>\n",
       "      <td>0.325</td>\n",
       "      <td>3</td>\n",
       "      <td>135512.093</td>\n",
       "      <td>188687.564</td>\n",
       "      <td>0.616</td>\n",
       "      <td>0.616</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca4_lr</th>\n",
       "      <td>0.218</td>\n",
       "      <td>0.544</td>\n",
       "      <td>4</td>\n",
       "      <td>129698.845</td>\n",
       "      <td>182623.042</td>\n",
       "      <td>0.652</td>\n",
       "      <td>0.652</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca5_lr</th>\n",
       "      <td>0.272</td>\n",
       "      <td>0.421</td>\n",
       "      <td>5</td>\n",
       "      <td>124516.303</td>\n",
       "      <td>177640.366</td>\n",
       "      <td>0.668</td>\n",
       "      <td>0.668</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca6_lr</th>\n",
       "      <td>0.260</td>\n",
       "      <td>0.443</td>\n",
       "      <td>6</td>\n",
       "      <td>124728.970</td>\n",
       "      <td>178142.141</td>\n",
       "      <td>0.668</td>\n",
       "      <td>0.668</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca7_lr</th>\n",
       "      <td>0.265</td>\n",
       "      <td>0.452</td>\n",
       "      <td>7</td>\n",
       "      <td>125056.235</td>\n",
       "      <td>178685.044</td>\n",
       "      <td>0.670</td>\n",
       "      <td>0.670</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca8_lr</th>\n",
       "      <td>0.268</td>\n",
       "      <td>0.453</td>\n",
       "      <td>8</td>\n",
       "      <td>124146.398</td>\n",
       "      <td>178264.453</td>\n",
       "      <td>0.676</td>\n",
       "      <td>0.676</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca9_lr</th>\n",
       "      <td>0.313</td>\n",
       "      <td>0.390</td>\n",
       "      <td>9</td>\n",
       "      <td>122758.238</td>\n",
       "      <td>174211.636</td>\n",
       "      <td>0.686</td>\n",
       "      <td>0.686</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unscaled_lr</th>\n",
       "      <td>0.382</td>\n",
       "      <td>0.435</td>\n",
       "      <td>113</td>\n",
       "      <td>89271.060</td>\n",
       "      <td>129586.400</td>\n",
       "      <td>0.829</td>\n",
       "      <td>0.829</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unscaled_noIntercept_lr</th>\n",
       "      <td>0.378</td>\n",
       "      <td>0.447</td>\n",
       "      <td>113</td>\n",
       "      <td>89314.802</td>\n",
       "      <td>130494.337</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.822</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           cv_mean_r2  \\\n",
       "norm_lr                 -440368777051548579528704.000   \n",
       "std_kBest10_lr                                  0.234   \n",
       "std_kBest15_lr                                  0.378   \n",
       "std_kBest20_lr                                  0.491   \n",
       "std_kBest25_lr                                  0.473   \n",
       "std_kBest30_lr                                  0.562   \n",
       "std_kBest35_lr                                  0.571   \n",
       "std_kBest40_lr                                  0.577   \n",
       "std_kBest45_lr                                  0.576   \n",
       "std_kBest5_lr                                   0.171   \n",
       "std_lr                    -3339445334256560111616.000   \n",
       "std_pca10_lr                                    0.329   \n",
       "std_pca1_lr                                    -0.362   \n",
       "std_pca2_lr                                    -0.017   \n",
       "std_pca3_lr                                     0.320   \n",
       "std_pca4_lr                                     0.218   \n",
       "std_pca5_lr                                     0.272   \n",
       "std_pca6_lr                                     0.260   \n",
       "std_pca7_lr                                     0.265   \n",
       "std_pca8_lr                                     0.268   \n",
       "std_pca9_lr                                     0.313   \n",
       "unscaled_lr                                     0.382   \n",
       "unscaled_noIntercept_lr                         0.378   \n",
       "\n",
       "                                           cv_std_r2  n_coeff        mae  \\\n",
       "norm_lr                 765316432812680247508992.000      113  89450.309   \n",
       "std_kBest10_lr                                 0.612       12 109669.750   \n",
       "std_kBest15_lr                                 0.513       17 107982.595   \n",
       "std_kBest20_lr                                 0.258       23 107677.423   \n",
       "std_kBest25_lr                                 0.272       28  98323.827   \n",
       "std_kBest30_lr                                 0.229       34  96460.662   \n",
       "std_kBest35_lr                                 0.236       40  95258.178   \n",
       "std_kBest40_lr                                 0.225       45  94291.720   \n",
       "std_kBest45_lr                                 0.243       51  94890.419   \n",
       "std_kBest5_lr                                  0.542        6 127609.832   \n",
       "std_lr                    4289501379982355070976.000      113  88737.220   \n",
       "std_pca10_lr                                   0.368       10 121423.807   \n",
       "std_pca1_lr                                    0.417        1 211408.806   \n",
       "std_pca2_lr                                    0.504        2 174395.700   \n",
       "std_pca3_lr                                    0.325        3 135512.093   \n",
       "std_pca4_lr                                    0.544        4 129698.845   \n",
       "std_pca5_lr                                    0.421        5 124516.303   \n",
       "std_pca6_lr                                    0.443        6 124728.970   \n",
       "std_pca7_lr                                    0.452        7 125056.235   \n",
       "std_pca8_lr                                    0.453        8 124146.398   \n",
       "std_pca9_lr                                    0.390        9 122758.238   \n",
       "unscaled_lr                                    0.435      113  89271.060   \n",
       "unscaled_noIntercept_lr                        0.447      113  89314.802   \n",
       "\n",
       "                              rmse    r2  score  is_accuracy_score  \\\n",
       "norm_lr                 129868.311 0.821  0.821              0.000   \n",
       "std_kBest10_lr          158603.304 0.736  0.736                nan   \n",
       "std_kBest15_lr          156396.261 0.742  0.742                nan   \n",
       "std_kBest20_lr          157238.313 0.750  0.750                nan   \n",
       "std_kBest25_lr          142234.782 0.786  0.786                nan   \n",
       "std_kBest30_lr          140511.012 0.793  0.793                nan   \n",
       "std_kBest35_lr          137727.173 0.803  0.803                nan   \n",
       "std_kBest40_lr          138799.275 0.794  0.794                nan   \n",
       "std_kBest45_lr          137744.109 0.800  0.800                nan   \n",
       "std_kBest5_lr           179204.935 0.660  0.660                nan   \n",
       "std_lr                  129438.012 0.826  0.826                nan   \n",
       "std_pca10_lr            171687.925 0.689  0.689                nan   \n",
       "std_pca1_lr             286644.592 0.139  0.139                nan   \n",
       "std_pca2_lr             240532.973 0.400  0.400                nan   \n",
       "std_pca3_lr             188687.564 0.616  0.616                nan   \n",
       "std_pca4_lr             182623.042 0.652  0.652                nan   \n",
       "std_pca5_lr             177640.366 0.668  0.668                nan   \n",
       "std_pca6_lr             178142.141 0.668  0.668                nan   \n",
       "std_pca7_lr             178685.044 0.670  0.670                nan   \n",
       "std_pca8_lr             178264.453 0.676  0.676                nan   \n",
       "std_pca9_lr             174211.636 0.686  0.686                nan   \n",
       "unscaled_lr             129586.400 0.829  0.829                nan   \n",
       "unscaled_noIntercept_lr 130494.337 0.822  0.822                nan   \n",
       "\n",
       "                         oos_accuracy_score  pca_explained_pct  \n",
       "norm_lr                               0.000                nan  \n",
       "std_kBest10_lr                          nan                nan  \n",
       "std_kBest15_lr                          nan                nan  \n",
       "std_kBest20_lr                          nan                nan  \n",
       "std_kBest25_lr                          nan                nan  \n",
       "std_kBest30_lr                          nan                nan  \n",
       "std_kBest35_lr                          nan                nan  \n",
       "std_kBest40_lr                          nan                nan  \n",
       "std_kBest45_lr                          nan                nan  \n",
       "std_kBest5_lr                           nan                nan  \n",
       "std_lr                                  nan                nan  \n",
       "std_pca10_lr                            nan              0.726  \n",
       "std_pca1_lr                             nan              0.194  \n",
       "std_pca2_lr                             nan              0.350  \n",
       "std_pca3_lr                             nan              0.436  \n",
       "std_pca4_lr                             nan              0.505  \n",
       "std_pca5_lr                             nan              0.559  \n",
       "std_pca6_lr                             nan              0.606  \n",
       "std_pca7_lr                             nan              0.637  \n",
       "std_pca8_lr                             nan              0.667  \n",
       "std_pca9_lr                             nan              0.697  \n",
       "unscaled_lr                             nan                nan  \n",
       "unscaled_noIntercept_lr                 nan                nan  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# build result dataframe\n",
    "dfCols = {}\n",
    "for k, (rc, rd) in linMdlRes.items():\n",
    "    dfCols[k] = rc\n",
    "pd.set_option('display.float_format', lambda x: '%.3f' % x)\n",
    "resDf = pd.DataFrame.from_dict(dfCols, orient='index')\n",
    "display(resDf)\n",
    "pd.set_option('display.float_format', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def model_result_df(mdlRes):\n",
    "#     # convert a dictionary of model results into a single dataframe with scalar properties\n",
    "#     copyCols = ['score', 'mae', 'rmse', 'r2']\n",
    "#     res = {}\n",
    "#     for k, md in mdlRes.items():        \n",
    "#         mres = {}\n",
    "#         for cc in copyCols:\n",
    "#             try:\n",
    "#                 mres[cc] = md[cc]\n",
    "#             except:\n",
    "#                 continue\n",
    "#         # not all models will have done cross-validation\n",
    "#         try:\n",
    "#             mres['n_coeff'] = md['coefs'].size\n",
    "#             mres['cv_mean_r2'] = md['cv_r2'].mean()\n",
    "#             mres['cv_std_r2'] = md['cv_r2'].std()        \n",
    "#         except:\n",
    "#             pass\n",
    "#         try:\n",
    "#             mres['pca_variance_ratio'] = md['pca_explained_pct']\n",
    "#         except:\n",
    "#             pass\n",
    "#         try:\n",
    "#             mres['accuracy_score'] = md['accuracy_score']\n",
    "#         except:\n",
    "#             pass\n",
    "#         res[k] = mres\n",
    "#     df = pd.DataFrame.from_dict(res, orient='index')\n",
    "#     return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>mae</th>\n",
       "      <th>rmse</th>\n",
       "      <th>r2</th>\n",
       "      <th>n_coeff</th>\n",
       "      <th>cv_mean_r2</th>\n",
       "      <th>cv_std_r2</th>\n",
       "      <th>pca_variance_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>norm_lr</th>\n",
       "      <td>0.708188</td>\n",
       "      <td>87817.515748</td>\n",
       "      <td>123914.491661</td>\n",
       "      <td>0.708188</td>\n",
       "      <td>20</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest2_lr</th>\n",
       "      <td>0.247254</td>\n",
       "      <td>152889.515566</td>\n",
       "      <td>198399.753236</td>\n",
       "      <td>0.247254</td>\n",
       "      <td>2</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest3_lr</th>\n",
       "      <td>0.288525</td>\n",
       "      <td>150130.270186</td>\n",
       "      <td>193704.272455</td>\n",
       "      <td>0.288525</td>\n",
       "      <td>3</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest4_lr</th>\n",
       "      <td>0.571736</td>\n",
       "      <td>110919.688656</td>\n",
       "      <td>150231.495281</td>\n",
       "      <td>0.571736</td>\n",
       "      <td>4</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest5_lr</th>\n",
       "      <td>0.626688</td>\n",
       "      <td>104221.106406</td>\n",
       "      <td>140099.709086</td>\n",
       "      <td>0.626688</td>\n",
       "      <td>5</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest6_lr</th>\n",
       "      <td>0.631156</td>\n",
       "      <td>102659.211680</td>\n",
       "      <td>139679.616267</td>\n",
       "      <td>0.631156</td>\n",
       "      <td>6</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest7_lr</th>\n",
       "      <td>0.634869</td>\n",
       "      <td>102322.866636</td>\n",
       "      <td>138537.827491</td>\n",
       "      <td>0.634869</td>\n",
       "      <td>7</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_kBest8_lr</th>\n",
       "      <td>0.625005</td>\n",
       "      <td>102417.640540</td>\n",
       "      <td>138429.518896</td>\n",
       "      <td>0.625005</td>\n",
       "      <td>8</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_lr</th>\n",
       "      <td>0.717707</td>\n",
       "      <td>86693.019554</td>\n",
       "      <td>121720.367746</td>\n",
       "      <td>0.717707</td>\n",
       "      <td>20</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca10_lr</th>\n",
       "      <td>0.608621</td>\n",
       "      <td>106022.456540</td>\n",
       "      <td>143426.971795</td>\n",
       "      <td>0.608621</td>\n",
       "      <td>10</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.927608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca1_lr</th>\n",
       "      <td>0.199196</td>\n",
       "      <td>161775.588452</td>\n",
       "      <td>207434.193097</td>\n",
       "      <td>0.199196</td>\n",
       "      <td>1</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.332354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca2_lr</th>\n",
       "      <td>0.205023</td>\n",
       "      <td>160150.945449</td>\n",
       "      <td>205861.339347</td>\n",
       "      <td>0.205023</td>\n",
       "      <td>2</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.493514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca3_lr</th>\n",
       "      <td>0.208579</td>\n",
       "      <td>158554.024413</td>\n",
       "      <td>204134.371980</td>\n",
       "      <td>0.208579</td>\n",
       "      <td>3</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.605383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca4_lr</th>\n",
       "      <td>0.372033</td>\n",
       "      <td>139420.858942</td>\n",
       "      <td>181526.697509</td>\n",
       "      <td>0.372033</td>\n",
       "      <td>4</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.677502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca5_lr</th>\n",
       "      <td>0.545194</td>\n",
       "      <td>116573.831930</td>\n",
       "      <td>155149.655009</td>\n",
       "      <td>0.545194</td>\n",
       "      <td>5</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.736176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca6_lr</th>\n",
       "      <td>0.540068</td>\n",
       "      <td>116142.857696</td>\n",
       "      <td>155139.313490</td>\n",
       "      <td>0.540068</td>\n",
       "      <td>6</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.793571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca7_lr</th>\n",
       "      <td>0.545688</td>\n",
       "      <td>115643.987415</td>\n",
       "      <td>154576.755856</td>\n",
       "      <td>0.545688</td>\n",
       "      <td>7</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.837845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca8_lr</th>\n",
       "      <td>0.543737</td>\n",
       "      <td>116215.103402</td>\n",
       "      <td>154680.374653</td>\n",
       "      <td>0.543737</td>\n",
       "      <td>8</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.875986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_pca9_lr</th>\n",
       "      <td>0.597259</td>\n",
       "      <td>105724.102129</td>\n",
       "      <td>145578.451752</td>\n",
       "      <td>0.597259</td>\n",
       "      <td>9</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>0.907929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_poly2_lr</th>\n",
       "      <td>0.801403</td>\n",
       "      <td>67802.880775</td>\n",
       "      <td>102316.124556</td>\n",
       "      <td>0.801403</td>\n",
       "      <td>231</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_poly3_lr</th>\n",
       "      <td>0.824917</td>\n",
       "      <td>62919.231190</td>\n",
       "      <td>96377.553261</td>\n",
       "      <td>0.824917</td>\n",
       "      <td>1771</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unscaled_lr</th>\n",
       "      <td>0.706821</td>\n",
       "      <td>87037.398941</td>\n",
       "      <td>122499.182573</td>\n",
       "      <td>0.706821</td>\n",
       "      <td>20</td>\n",
       "      <td>0.253639</td>\n",
       "      <td>0.333506</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unscaled_noIntercept_lr</th>\n",
       "      <td>0.704990</td>\n",
       "      <td>87071.426325</td>\n",
       "      <td>124721.716353</td>\n",
       "      <td>0.704990</td>\n",
       "      <td>20</td>\n",
       "      <td>0.351695</td>\n",
       "      <td>0.202927</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            score            mae           rmse        r2  \\\n",
       "norm_lr                  0.708188   87817.515748  123914.491661  0.708188   \n",
       "std_kBest2_lr            0.247254  152889.515566  198399.753236  0.247254   \n",
       "std_kBest3_lr            0.288525  150130.270186  193704.272455  0.288525   \n",
       "std_kBest4_lr            0.571736  110919.688656  150231.495281  0.571736   \n",
       "std_kBest5_lr            0.626688  104221.106406  140099.709086  0.626688   \n",
       "std_kBest6_lr            0.631156  102659.211680  139679.616267  0.631156   \n",
       "std_kBest7_lr            0.634869  102322.866636  138537.827491  0.634869   \n",
       "std_kBest8_lr            0.625005  102417.640540  138429.518896  0.625005   \n",
       "std_lr                   0.717707   86693.019554  121720.367746  0.717707   \n",
       "std_pca10_lr             0.608621  106022.456540  143426.971795  0.608621   \n",
       "std_pca1_lr              0.199196  161775.588452  207434.193097  0.199196   \n",
       "std_pca2_lr              0.205023  160150.945449  205861.339347  0.205023   \n",
       "std_pca3_lr              0.208579  158554.024413  204134.371980  0.208579   \n",
       "std_pca4_lr              0.372033  139420.858942  181526.697509  0.372033   \n",
       "std_pca5_lr              0.545194  116573.831930  155149.655009  0.545194   \n",
       "std_pca6_lr              0.540068  116142.857696  155139.313490  0.540068   \n",
       "std_pca7_lr              0.545688  115643.987415  154576.755856  0.545688   \n",
       "std_pca8_lr              0.543737  116215.103402  154680.374653  0.543737   \n",
       "std_pca9_lr              0.597259  105724.102129  145578.451752  0.597259   \n",
       "std_poly2_lr             0.801403   67802.880775  102316.124556  0.801403   \n",
       "std_poly3_lr             0.824917   62919.231190   96377.553261  0.824917   \n",
       "unscaled_lr              0.706821   87037.398941  122499.182573  0.706821   \n",
       "unscaled_noIntercept_lr  0.704990   87071.426325  124721.716353  0.704990   \n",
       "\n",
       "                         n_coeff  cv_mean_r2  cv_std_r2  pca_variance_ratio  \n",
       "norm_lr                       20    0.253639   0.333506                 NaN  \n",
       "std_kBest2_lr                  2    0.253639   0.333506                 NaN  \n",
       "std_kBest3_lr                  3    0.253639   0.333506                 NaN  \n",
       "std_kBest4_lr                  4    0.253639   0.333506                 NaN  \n",
       "std_kBest5_lr                  5    0.253639   0.333506                 NaN  \n",
       "std_kBest6_lr                  6    0.253639   0.333506                 NaN  \n",
       "std_kBest7_lr                  7    0.253639   0.333506                 NaN  \n",
       "std_kBest8_lr                  8    0.253639   0.333506                 NaN  \n",
       "std_lr                        20    0.253639   0.333506                 NaN  \n",
       "std_pca10_lr                  10    0.253639   0.333506            0.927608  \n",
       "std_pca1_lr                    1    0.253639   0.333506            0.332354  \n",
       "std_pca2_lr                    2    0.253639   0.333506            0.493514  \n",
       "std_pca3_lr                    3    0.253639   0.333506            0.605383  \n",
       "std_pca4_lr                    4    0.253639   0.333506            0.677502  \n",
       "std_pca5_lr                    5    0.253639   0.333506            0.736176  \n",
       "std_pca6_lr                    6    0.253639   0.333506            0.793571  \n",
       "std_pca7_lr                    7    0.253639   0.333506            0.837845  \n",
       "std_pca8_lr                    8    0.253639   0.333506            0.875986  \n",
       "std_pca9_lr                    9    0.253639   0.333506            0.907929  \n",
       "std_poly2_lr                 231    0.253639   0.333506                 NaN  \n",
       "std_poly3_lr                1771    0.253639   0.333506                 NaN  \n",
       "unscaled_lr                   20    0.253639   0.333506                 NaN  \n",
       "unscaled_noIntercept_lr       20    0.351695   0.202927                 NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "linMdlDf = model_result_df(linMdlRes)\n",
    "display(linMdlDf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the results above, applying standard scaling to the linear regression model with the full set of features included does not meaningfully improve performance. None of these results look particularly good, however.\n",
    "\n",
    "As you increase the number of principal components, the r2 performance begins to converge towards the full set of predictors.\n",
    "\n",
    "The cubic polynomial regression performed best of the models attempted here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification\n",
    "For the classification algorithms, we will attempt to predict whether a condo unit is owner-occupied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: separate runner function for classification problems\n",
    "# predict whether a house is owner-occupied; it's a binary problem but you can still use SVM etc.\n",
    "targetCol = 'OWN_OCC'\n",
    "# create copy of pred for use in regressions, w/ only continuous\n",
    "contData = pred[regCont + [targetCol]].dropna(how='any').copy()\n",
    "# separate out target with aligned NaN removals\n",
    "contTgt = contData[targetCol]\n",
    "contData.drop(targetCol, axis=1, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "clsMdls = {}\n",
    "# logistic regression variants\n",
    "clsMdls['raw_logistic'] = LogisticRegression(solver='lbfgs')\n",
    "clsMdls['std_logistic'] = make_pipeline(StandardScaler(), \n",
    "                                        LogisticRegression(solver='lbfgs'))\n",
    "clsMdls['raw_mult_logistic'] = LogisticRegression(solver='lbfgs', multi_class='multinomial')\n",
    "clsMdls['std_mult_logistic'] = make_pipeline(StandardScaler(),\n",
    "                                             LogisticRegression(solver='lbfgs', multi_class='multinomial'))\n",
    "clsMdls['raw_decision_tree'] = DecisionTreeClassifier()\n",
    "clsMdls['std_decision_tree'] = make_pipeline(StandardScaler(),\n",
    "                                            DecisionTreeClassifier())\n",
    "clsMdls['raw_linear_svm'] = SVC(kernel='linear')\n",
    "clsMdls['raw_NB'] = GaussianNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'fitting raw_linear_svm'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clsMdlRes = {}\n",
    "for k, mdl in list(clsMdls.items()):\n",
    "    clear_output(wait=True)\n",
    "    display('fitting {}'.format(k))\n",
    "    clsMdlRes[k] = run_model(mdl, contData, contTgt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>raw_decision_tree</th>\n",
       "      <td>0.603884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>raw_logistic</th>\n",
       "      <td>0.642236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>raw_mult_logistic</th>\n",
       "      <td>0.642793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_decision_tree</th>\n",
       "      <td>0.608687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_logistic</th>\n",
       "      <td>0.668476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std_mult_logistic</th>\n",
       "      <td>0.664579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      score\n",
       "raw_decision_tree  0.603884\n",
       "raw_logistic       0.642236\n",
       "raw_mult_logistic  0.642793\n",
       "std_decision_tree  0.608687\n",
       "std_logistic       0.668476\n",
       "std_mult_logistic  0.664579"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clsMdlDf = model_result_df(clsMdlRes)\n",
    "display(clsMdlDf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering\n",
    "For clustering analysis, we will look at identifying similarities between Boston zip codes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# review zipcode data\n",
    "zipData.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}