Mecklenburg_County_Extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67e59447",
   "metadata": {},
   "outputs": [],
   "source": [
    "import osmium as osm\n",
    "import pandas as pd\n",
    "import geopandas\n",
    "import os.path\n",
    "import shapely.wkb as wkblib\n",
    "wkbfab = osm.geom.WKBFactory()\n",
    "import numpy as np\n",
    "import shapely.geometry\n",
    "\n",
    "from sklearn import tree\n",
    "import graphviz\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier, export_graphviz\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn import preprocessing\n",
    "import jenkspy\n",
    "from sklearn.metrics import PrecisionRecallDisplay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5482a8c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AreaHandler(osm.SimpleHandler):\n",
    "    def __init__(self):\n",
    "        osm.SimpleHandler.__init__(self)\n",
    "        self.num_polys = 0\n",
    "        self.osm_data = []\n",
    "        self.roads_data = []\n",
    "        self.landuse_data = []\n",
    "        self.amenity_data = []\n",
    "        \n",
    "    def count_polygons(self):\n",
    "        self.num_polys += 1\n",
    "        \n",
    "    def way(self, w):\n",
    "        try:\n",
    "            wkb = wkbfab.create_linestring(w)\n",
    "            line = wkblib.loads(wkb, hex=True)\n",
    "            for tag in w.tags:\n",
    "                if (tag.k=='highway'):\n",
    "                    self.roads_data.append([w.id,\n",
    "                                       tag.v,\n",
    "                                       line])\n",
    "        except Exception:\n",
    "            pass    \n",
    "        \n",
    "    def area(self, a):\n",
    "        try:\n",
    "            wkb = wkbfab.create_multipolygon(a)\n",
    "            self.count_polygons()\n",
    "            # load into shapely\n",
    "            poly = wkblib.loads(wkb, hex=True)\n",
    "            for tag in a.tags:\n",
    "                if (tag.k=='building'):\n",
    "                    self.osm_data.append([a.id,                                   \n",
    "                                    tag.v,\n",
    "                                    list(poly.geoms)[0],\n",
    "                                    list(dict(a.tags).keys()),\n",
    "                                    dict(a.tags)])\n",
    "                if (tag.k=='landuse'):\n",
    "                    self.landuse_data.append([tag.v,\n",
    "                                    list(poly.geoms)[0],\n",
    "                                    ])\n",
    "                if (tag.k == 'amenity'):\n",
    "                    self.amenity_data.append([tag.v,\n",
    "                                    list(poly.geoms)[0],\n",
    "                                    ])\n",
    "        except Exception:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23e304ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load official buildings\n",
    "buildings_path = \"./Mecklenburg_admin_buildings/Mecklenburg_admin_buildings.shp\"\n",
    "official_buildings = geopandas.read_file(buildings_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35282a22",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4994b815",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38ede48d",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings = official_buildings.rename(columns={'descproper': 'official_type',\\\n",
    "                                                        'objectid': 'OBJECTID'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68a9c9f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbe76be9",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9dca6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_list = ['Warehouse Lg','StadiumArena','Manufactured']\n",
    "official_buildings.dropna(inplace=True)\n",
    "official_buildings.drop(official_buildings[(official_buildings['official_type'].isin(drop_list))].index,\\\n",
    "                        inplace = True)\n",
    "official_buildings.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "013747c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d376cf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24ac1837",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2db9d5b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings['official_type'] = official_buildings['official_type'].apply(\\\n",
    "                                lambda x: 'RES' if (x == 'Condo/Townhome' \\\n",
    "                                                or x == 'Multi-Family'\\\n",
    "                                                or x == 'Single-Family'\n",
    "                                                ) else 'NON_RES')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56eef36d",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2469cca5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a0361f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "areahandler = AreaHandler()\n",
    "areahandler.apply_file(\"./Datasets/OSM/Mecklenburg_County_Buildings.osm.pbf\", locations=True)\n",
    "print(\"Number of polygons: %d\" % areahandler.num_polys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b86c913",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(areahandler.osm_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bab5ccd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the list into a pandas DataFrame\n",
    "data_colnames = ['id', 'building', 'geometry', 'all_tag_keys', 'all_tags']\n",
    "df = pd.DataFrame(areahandler.osm_data, columns=data_colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b901865",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a69c3029",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf = geopandas.GeoDataFrame(df, crs=4326, geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa0dc193",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bd58e1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "831733d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.groupby('building').count().sort_values(by=['building'], inplace=False, ascending=True).tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1af7265a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d60d34c3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea68123a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the list into a pandas DataFrame\n",
    "land_colnames = ['landuse', 'geometry']\n",
    "land_df = pd.DataFrame(areahandler.landuse_data, columns=land_colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cb4e6b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "343e1999",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d027da2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_df.groupby('landuse').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40a5f940",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_res = ['commercial', 'retail','industrial','grass','plant_nursery','quarry','railway','government',\\\n",
    "            'institutional']\n",
    "land_df['landuse'] = land_df['landuse'].apply(\\\n",
    "                                lambda x: 'non_res' if x in non_res else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "193ceef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_df.groupby('landuse').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c537f702",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_gdf = geopandas.GeoDataFrame(land_df, crs=4326, geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90869f59",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee64f53a",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c830abd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land = gdf.sjoin(land_gdf, how=\"left\", predicate='intersects')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac908859",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e51920e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.drop_duplicates(subset='id',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaea46b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f693c93e",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf02b276",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.rename(columns={'index_right': 'index_landuse'},inplace=True)\n",
    "buildings_land['landuse'] = \\\n",
    "            buildings_land['landuse'].apply(lambda x: 'no_landuse' if pd.isnull(x) else x)\n",
    "buildings_land.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62f56365",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land[buildings_land['landuse'] == 'no_landuse'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb50fff5",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(buildings_land.index_landuse.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7abcf39e",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc3147aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby('landuse').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb1c611c",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby(['landuse','building']).count().head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1ab27bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a6925da",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10e83674",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.crs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6aa5e376",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.to_crs(epsg=4326, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f82204bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf = buildings_land.sjoin(official_buildings, how=\"inner\", predicate='intersects')\n",
    "buildings_gdf.rename(columns={'index_right': 'index_official'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "255a8adb",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "093b6774",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d5c422b",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf.drop_duplicates(subset='id',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0385fac8",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebc8b78c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4b70dc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(areahandler.roads_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b57dba89",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the list of roads into a pandas DataFrame\n",
    "roads_colnames = ['id', 'highway', 'geometry']\n",
    "roads_df = pd.DataFrame(areahandler.roads_data, columns=roads_colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c139b63",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89ce21e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91296714",
   "metadata": {},
   "outputs": [],
   "source": [
    "road_categories = roads_df.groupby(['highway'])['id'].count().reset_index(name=\"count\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d51b7750",
   "metadata": {},
   "outputs": [],
   "source": [
    "road_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fc5712d",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1 = roads_df[(roads_df['highway'] == 'residential') | (roads_df['highway'] == 'living_street')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83d0adc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1 = roads_cat1.assign(highway='residential')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f3d92c",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85825be5",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1.groupby('highway').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56b301fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1 = geopandas.GeoDataFrame(roads_cat1,crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5530b297",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_list = ['primary', 'secondary','tertiary']\n",
    "roads_cat2 = roads_df[roads_df['highway'].isin(roads_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a926940e",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2 = roads_cat2.assign(highway='non_res_road')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62402081",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf52ab86",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2.groupby('highway').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "087e95f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2 = geopandas.GeoDataFrame(roads_cat2,crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdbea8a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3 = roads_df[(roads_df['highway'] == 'motorway') | (roads_df['highway'] == 'trunk')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7081dca",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3 = roads_cat3.assign(highway='roads_cat3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b271218c",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e95cd42",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3.groupby('highway').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "171879b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3 = geopandas.GeoDataFrame(roads_cat3,crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4254b99",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4 = roads_df[(roads_df['highway'] == 'service')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33f121b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30b3e640",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4.groupby('highway').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "140693fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4 = geopandas.GeoDataFrame(roads_cat4,crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84e0e945",
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_buffers(radius, road_category):\n",
    "    road_category['geometry'] = road_category['geometry'].to_crs(epsg=32610).buffer(radius)\n",
    "    road_category.to_crs(epsg=4326, inplace=True)\n",
    "    road_category.rename(columns={'highway': 'buffered_highway'}, inplace=True)\n",
    "    road_category = road_category.dissolve()\n",
    "    \n",
    "    return road_category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "929ce362",
   "metadata": {},
   "outputs": [],
   "source": [
    "# approx in meters unit: 1 decimal degree equals 111.32 km \n",
    "#radius_list = [0.00027,0.00054,0.00081]\n",
    "radius_list = [30,60,90]                        # meters\n",
    "road_category = roads_cat1.copy()\n",
    "buffered_cat1_30 = apply_buffers(radius_list[0],road_category)\n",
    "buffered_cat1_60 = apply_buffers(radius_list[1],road_category)\n",
    "buffered_cat1_90 = apply_buffers(radius_list[2],road_category)\n",
    "\n",
    "road_category = roads_cat2.copy()\n",
    "buffered_cat2_30 = apply_buffers(radius_list[0],road_category)\n",
    "buffered_cat2_60 = apply_buffers(radius_list[1],road_category)\n",
    "buffered_cat2_90 = apply_buffers(radius_list[2],road_category)\n",
    "\n",
    "road_category = roads_cat3.copy()\n",
    "buffered_cat3_30 = apply_buffers(radius_list[0],road_category)\n",
    "buffered_cat3_60 = apply_buffers(radius_list[1],road_category)\n",
    "buffered_cat3_90 = apply_buffers(radius_list[2],road_category)\n",
    "\n",
    "road_category = roads_cat4.copy()\n",
    "buffered_cat4_30 = apply_buffers(radius_list[0],road_category)\n",
    "buffered_cat4_60 = apply_buffers(radius_list[1],road_category)\n",
    "buffered_cat4_90 = apply_buffers(radius_list[2],road_category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a49ef419",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1.plot()\n",
    "buffered_cat1_30.plot()\n",
    "buffered_cat1_60.plot()\n",
    "buffered_cat1_90.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d476b714",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2.plot()\n",
    "buffered_cat2_30.plot()\n",
    "buffered_cat2_60.plot()\n",
    "buffered_cat2_90.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9238492",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3.plot()\n",
    "buffered_cat3_30.plot()\n",
    "buffered_cat3_60.plot()\n",
    "buffered_cat3_90.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08959842",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4.plot()\n",
    "buffered_cat4_30.plot()\n",
    "buffered_cat4_60.plot()\n",
    "buffered_cat4_90.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29b9ddd0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e966c67",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34ecd949",
   "metadata": {},
   "outputs": [],
   "source": [
    "def join_buildings(buffer_category, roads_category):\n",
    "    buffered_category = {\n",
    "        'cat1_30': buffered_cat1_30,\n",
    "        'cat1_60': buffered_cat1_60,\n",
    "        'cat1_90': buffered_cat1_90,\n",
    "        'cat2_30': buffered_cat2_30,\n",
    "        'cat2_60': buffered_cat2_60,\n",
    "        'cat2_90': buffered_cat2_90,\n",
    "        'cat3_30': buffered_cat3_30,\n",
    "        'cat3_60': buffered_cat3_60,\n",
    "        'cat3_90': buffered_cat3_90,\n",
    "        'cat4_30': buffered_cat4_30,\n",
    "        'cat4_60': buffered_cat4_60,\n",
    "        'cat4_90': buffered_cat4_90,\n",
    "\n",
    "    }.get(buffer_category)\n",
    "\n",
    "    joined = buildings_gdf.sjoin(buffered_category, how='left')\n",
    "    #joined = buildings_gdf.sjoin(buffered_category, how=\"left\", predicate='intersects')\n",
    "    joined.rename(columns={'index_right': 'index_roads'}, inplace=True)\n",
    "    #joined.drop_duplicates(subset='id_left',inplace=True)\n",
    "    if roads_category:\n",
    "        joined['buffered_highway'] = \\\n",
    "            joined['buffered_highway'].apply(lambda x: 'not_residential' if pd.isnull(x) else x)\n",
    "    else:\n",
    "        joined['buffered_highway'] = \\\n",
    "            joined['buffered_highway'].apply(lambda x: 'residential' if pd.isnull(x) else x)\n",
    "\n",
    "    joined.reset_index(inplace=True, drop=True)\n",
    "    \n",
    "    return joined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a0609f0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29cd4e4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "buffer_categories = ['cat1_30', 'cat1_60', 'cat1_90','cat2_30','cat2_60','cat2_90',\\\n",
    "                    'cat3_30', 'cat3_60', 'cat3_90','cat4_30','cat4_60','cat4_90']\n",
    "roads_category = 1                      # 1 is residential, 0 is ['primary', 'secondary','tertiary']\n",
    "joined_buildings1 = join_buildings(buffer_categories[0],roads_category)\n",
    "joined_buildings2 = join_buildings(buffer_categories[1],roads_category)\n",
    "joined_buildings3 = join_buildings(buffer_categories[2],roads_category)\n",
    "roads_category = 0\n",
    "joined_buildings4 = join_buildings(buffer_categories[3],roads_category)\n",
    "joined_buildings5 = join_buildings(buffer_categories[4],roads_category)\n",
    "joined_buildings6 = join_buildings(buffer_categories[5],roads_category)\n",
    "joined_buildings7 = join_buildings(buffer_categories[6],roads_category)\n",
    "joined_buildings8 = join_buildings(buffer_categories[7],roads_category)\n",
    "joined_buildings9 = join_buildings(buffer_categories[8],roads_category)\n",
    "joined_buildings10 = join_buildings(buffer_categories[9],roads_category)\n",
    "joined_buildings11 = join_buildings(buffer_categories[10],roads_category)\n",
    "joined_buildings12 = join_buildings(buffer_categories[11],roads_category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b2e0b5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(joined_buildings1.shape)\n",
    "print(joined_buildings2.shape)\n",
    "print(joined_buildings3.shape)\n",
    "print(joined_buildings4.shape)\n",
    "print(joined_buildings5.shape)\n",
    "print(joined_buildings6.shape)\n",
    "print(joined_buildings7.shape)\n",
    "print(joined_buildings8.shape)\n",
    "print(joined_buildings9.shape)\n",
    "print(joined_buildings10.shape)\n",
    "print(joined_buildings11.shape)\n",
    "print(joined_buildings12.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c3655dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings1.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "478458f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings1.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce21550c",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings2.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df180915",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings3.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "730ff030",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings4.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "065fab87",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings5.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98041d65",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings6.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65292d21",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings7.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29776bf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings8.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5b04d6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings9.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aee2ab12",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6310c4d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(areahandler.osm_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de947b83",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_list = []\n",
    "for sublist in areahandler.osm_data:\n",
    "    keys_list.append(sublist[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "445f784f",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_list_flat = [val for sublist in keys_list for val in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "425c3ba3",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_keys = sorted(list(set(keys_list_flat)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f52bf4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e7fe704",
   "metadata": {},
   "outputs": [],
   "source": [
    "key_freq = []\n",
    "for key in unique_keys:\n",
    "    key_freq.append(keys_list_flat.count(key))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ed3c2f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(key_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d11ad262",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_tuples = list(zip(unique_keys, key_freq))\n",
    "keys_freq_df = pd.DataFrame(keys_tuples, columns = ['Key', 'Freq'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "489b0dfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.sort_values(by=['Freq'], inplace=True, ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b2f24ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8fedc34",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5dc6fd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.sort_values(by=['Key'], inplace=False, ascending=True).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca41dc73",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.head(21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cd0ad78",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7fd7083",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_tags = ['source','addr:street','name','building:levels',\\\n",
    "                'amenity','brand','website','shop']\n",
    "num_keys = len(selected_tags)+2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71d0a6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(selected_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea139454",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40c9e15e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(input_data, value_tags, selected_tags):\n",
    "    col_pos = 2\n",
    "    for tag_key in selected_tags:\n",
    "        col_name = tag_key\n",
    "        input_data.insert(col_pos,col_name,0)\n",
    "        input_data[col_name] = input_data.apply(lambda x: 1 if tag_key in list(x['all_tag_keys']) \\\n",
    "            else 0, axis=1)\n",
    "\n",
    "    for tag_key in value_tags:\n",
    "        input_data[tag_key] = input_data.apply(lambda x: x['all_tags'].get(tag_key) \\\n",
    "            if tag_key in list(x['all_tag_keys']) else 0, axis=1)\n",
    "\n",
    "    # fix anomalies in FFX County data\n",
    "    input_data.drop(input_data[(input_data['building:levels'] == 'K-6')].index, inplace=True)\n",
    "    input_data.loc[(input_data['building:levels'] == '-1,1,2,3'),'building:levels']=4\n",
    "\n",
    "    input_data.drop(columns=['all_tag_keys', 'all_tags'], inplace=True)\n",
    "\n",
    "    input_data['official_type'] = \\\n",
    "    input_data['official_type'].apply(lambda x: 1 if x == 'RES' else 0)\n",
    "\n",
    "    return input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64f6052e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_buffers(input_data,col_names, buffers_list):\n",
    "    for num in range(0, len(col_names)):\n",
    "        input_data.insert(num+2,col_names[num],buffers_list[num])\n",
    "        input_data[col_names[num]] = input_data[col_names[num]].apply(lambda x: 1 if x == 'residential' else 0)\n",
    "\n",
    "    return input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbde3091",
   "metadata": {},
   "outputs": [],
   "source": [
    "#num_buffers = 6\n",
    "#buffer_loc = 0\n",
    "#value_tags = ['roof:shape','building:levels']\n",
    "#col_names = ['buffer_cat1_30','buffer_cat1_60','buffer_cat1_90',\\\n",
    "#            'buffer_cat2_30','buffer_cat2_60','buffer_cat2_90']\n",
    "#buffers_list = [joined_buildings1['buffered_highway'],joined_buildings2['buffered_highway'],\\\n",
    "#                joined_buildings3['buffered_highway'],joined_buildings4['buffered_highway'],\\\n",
    "#                joined_buildings5['buffered_highway'],joined_buildings6['buffered_highway']]\n",
    "\n",
    "num_buffers = 12\n",
    "buffer_loc = 0\n",
    "value_tags = ['building:levels']\n",
    "col_names = ['buffer_cat1_30','buffer_cat1_60','buffer_cat1_90',\\\n",
    "            'buffer_cat2_30','buffer_cat2_60','buffer_cat2_90',\\\n",
    "            'buffer_cat3_30','buffer_cat3_60','buffer_cat3_90',\\\n",
    "            'buffer_cat4_30','buffer_cat4_60','buffer_cat4_90']\n",
    "buffers_list = [joined_buildings1['buffered_highway'],joined_buildings2['buffered_highway'],\\\n",
    "                joined_buildings3['buffered_highway'],joined_buildings4['buffered_highway'],\\\n",
    "                joined_buildings5['buffered_highway'],joined_buildings6['buffered_highway'],\\\n",
    "                joined_buildings7['buffered_highway'],joined_buildings8['buffered_highway'],\\\n",
    "                joined_buildings9['buffered_highway'],joined_buildings10['buffered_highway'],\\\n",
    "                joined_buildings11['buffered_highway'],joined_buildings12['buffered_highway']]\n",
    "backup_data = joined_buildings1.copy()\n",
    "#input_data = input_data[['geometry','official_type','land_building','all_tag_keys','all_tags']]\n",
    "#input_data = input_data[['geometry','official_type','building','landuse','all_tag_keys','all_tags']]\n",
    "#input_data = input_data[['geometry','official_type','building','landuse',\\\n",
    "#                        'all_tag_keys','all_tags','id_left']]\n",
    "#input_data = input_data[['geometry','official_type','building','landuse','amenity_key','parking_area',\\\n",
    "#                        'all_tag_keys','all_tags']]\n",
    "backup_data = backup_data[['geometry','official_type','building','landuse',\\\n",
    "                        'all_tag_keys','all_tags','id_left']]\n",
    "backup_data.insert(len(backup_data.columns), 'area',backup_data['geometry'].to_crs(epsg=32610).area)\n",
    "backup_data = preprocess_data(backup_data,value_tags,selected_tags)\n",
    "\n",
    "backup_data = add_buffers(backup_data,col_names[buffer_loc:buffer_loc+num_buffers],\\\n",
    "                         buffers_list[buffer_loc:buffer_loc+num_buffers])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45beccc6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "335289bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(areahandler.amenity_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5223c00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the list of roads into a pandas DataFrame\n",
    "amenity_colnames = ['amenity_key', 'geometry']\n",
    "amenity_df = pd.DataFrame(areahandler.amenity_data, columns=amenity_colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "755894bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "amenity_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7537d3e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "amenity_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1d883de",
   "metadata": {},
   "outputs": [],
   "source": [
    "amenity_categories = amenity_df.groupby(['amenity_key'])['geometry'].count().reset_index(name=\"count\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29fb4888",
   "metadata": {},
   "outputs": [],
   "source": [
    "amenity_categories.tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9785f94",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_df = amenity_df[(amenity_df['amenity_key'] == 'parking') | (amenity_df['amenity_key'] == 'parking_space')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c826a14f",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_df = parking_df.assign(amenity_key='parking')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d57ce083",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23d0198f",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "752fb249",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf = geopandas.GeoDataFrame(parking_df.copy(),crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b492bc0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.insert(len(parking_gdf.columns), 'parking_area',parking_gdf['geometry'].to_crs(epsg=32610).area)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16e4c539",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dfddbfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "feb830f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "hist = parking_gdf.hist(bins=100, grid=False, figsize=(12,8), color='#86bf91', zorder=2, rwidth=0.9)\n",
    "plt.yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b8d0fdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "breaks = jenkspy.jenks_breaks(parking_gdf['parking_area'], nb_class=3)\n",
    "print(breaks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efce7a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins_list = ['bin1','bin2','bin3']\n",
    "parking_gdf['cut_jenks'] = pd.cut(parking_gdf['parking_area'],\n",
    "                                bins=breaks,\n",
    "                                labels=bins_list,\n",
    "                                include_lowest=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6d60c49",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4650f7d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "for bin in bins_list:\n",
    "    parking_gdf.insert(len(parking_gdf.columns),bin,0)\n",
    "    parking_gdf[bin] = parking_gdf.apply(lambda x: 1 if x['cut_jenks'] == bin\\\n",
    "                                     else 0, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "264a9047",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73bb2f0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1a993e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "radius_list = [30,60,90]\n",
    "counter = 1\n",
    "for bin in bins_list:\n",
    "    for radius in radius_list:\n",
    "        parking_buffer = parking_gdf.copy()\n",
    "        parking_buffer = parking_buffer[['geometry',bin]]\n",
    "        col_name = 'parking_cat'+str(counter)\n",
    "        counter += 1\n",
    "        parking_buffer.rename(columns={bin: col_name}, inplace=True)\n",
    "        parking_buffer['geometry'] = parking_buffer['geometry'].to_crs(epsg=32610).buffer(radius)\n",
    "        parking_buffer.to_crs(epsg=4326, inplace=True)\n",
    "        buildings_parking = backup_data.sjoin(parking_buffer, how=\"left\", predicate='intersects')\n",
    "        buildings_parking[col_name] = buildings_parking[col_name].fillna(0)\n",
    "        buildings_parking.drop_duplicates(subset='id_left',inplace=True)\n",
    "        buildings_parking.reset_index(inplace=True, drop=True)\n",
    "        backup_data.insert(len(backup_data.columns),col_name,list(buildings_parking[col_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65e97f6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.drop(columns=['id_left'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f165b58b",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7559fb24",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f84c5abb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#building:levels = 1,2,3\n",
    "backup_data.loc[(backup_data['building:levels'] == '1,2,3'),'building:levels']=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b03f9f7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#backup_data.to_pickle(\"./Mecklenburg_Backup_Data/Mecklenburg.pkl\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e60b4e3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#backup_data = pd.read_pickle(\"./Mecklenburg_Backup_Data/Mecklenburg.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cabd7d4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data = backup_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0ab28b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30dc341f",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7758f1f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('landuse').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ef89d72",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2fac583",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('building').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4c2d553",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['building'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd1a068e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(input_data['building'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "582bfe3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "FFX_building_features = np.array(['airport_terminal', 'yes', 'office', 'retail', 'no', 'school',\n",
    "       'industrial', 'commercial', 'roof', 'hangar', 'museum',\n",
    "       'apartments', 'garage', 'residential', 'parking', 'house',\n",
    "       'train_station', 'warehouse', 'hospital', 'canopy', 'church',\n",
    "       'data_center', 'kindergarten', 'terrace', 'boathouse', 'service',\n",
    "       'detached', 'attached', 'public', 'supermarket', 'barn',\n",
    "       'pavilion', 'sports_centre', 'mosque', 'gazebo', 'hotel',\n",
    "       'fire_station', 'central_office', 'grandstand', 'bunker',\n",
    "       'government', 'office;yes', 'shed', 'college', 'construction',\n",
    "       'chapel', 'civic', 'transportation', 'bridge', 'storage',\n",
    "       'semidetached_house', 'garages', 'greenhouse', 'manufacture',\n",
    "       'commecial', 'turning_circle', 'q', 'dormitory', 'townhome',\n",
    "       'Pumping_Station', 'townhouse', 'Townhouse', 'prison',\n",
    "       'foundation', 'cabin', 'static_caravan', 'religious'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28dbb4c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(FFX_building_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4e3e5ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_building_features = np.array(['dormitory', 'office', 'yes', 'university', 'church', 'retail',\n",
    "       'government', 'sports_centre', 'garage', 'stadium', 'school',\n",
    "       'public', 'industrial', 'apartments', 'amenity', 'commercial',\n",
    "       'house', 'roof', 'residential', 'garages', 'shed', 'hotel',\n",
    "       'detached', 'warehouse', 'museum', 'parking', 'construction',\n",
    "       'greenhouse', 'terrace', 'hospital', 'civic', 'semidetached_house',\n",
    "       'service', 'static_caravan', 'kindergarten', 'kiosk', 'carport'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aff5b397",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(Boulder_building_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "533ce901",
   "metadata": {},
   "outputs": [],
   "source": [
    "MLB_FFX = np.intersect1d(input_data['building'].unique(), FFX_building_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ccb8915",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(MLB_FFX.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "210a94e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "common_buildings = np.intersect1d(MLB_FFX, Boulder_building_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9eb1e35",
   "metadata": {},
   "outputs": [],
   "source": [
    "common_buildings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbd3d354",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(common_buildings.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d77ee6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_buildings = np.setxor1d(input_data['building'].unique(), common_buildings).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8e00748",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_buildings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85982bdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data[input_data['building'].isin(unique_buildings)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "509c56ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['building'] = input_data['building'].apply(\\\n",
    "                                lambda x: 'misc_buildings' if x in(unique_buildings)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5c20cc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data[input_data['building'].isin(unique_buildings)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a9029e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8272124f",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('building').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c42ed013",
   "metadata": {},
   "outputs": [],
   "source": [
    "common_landuse = np.array(['non_res', 'no_landuse', 'residential', 'construction',\n",
    "       'forest', 'recreation_ground', 'cemetery', 'farmyard'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ec3e842",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_landuse = np.setxor1d(input_data['landuse'].unique(), common_landuse).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96009d26",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_landuse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01aa979e",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data[input_data['landuse'].isin(unique_landuse)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "288ca632",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['landuse'] = input_data['landuse'].apply(\\\n",
    "                                lambda x: 'misc_landuse' if x in(unique_landuse)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5e2b793",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data[input_data['landuse'].isin(unique_landuse)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2fda725",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('landuse').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b834eb89",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['landuse'].replace(['residential','non_res','forest','cemetery','recreation_ground',\\\n",
    "                                'construction','farmyard'],['residential_landuse','non_res_landuse',\\\n",
    "                                'forest_landuse','cemetery_landuse','recreation_ground_landuse',\\\n",
    "                                'construction_landuse','farmyard_landuse'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24ef5bae",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('landuse').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26b907a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de87d8a9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4d90b5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline = input_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "125eba91",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "191b3f05",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline = Mecklenburg_baseline[['geometry', 'official_type', 'building']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "288e868b",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c8efb25",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "735e3569",
   "metadata": {},
   "outputs": [],
   "source": [
    "residential_list = ['residential','apartments','dormitory', 'house','semidetached_house']\n",
    "Mecklenburg_baseline['building'] = Mecklenburg_baseline['building'].apply(\\\n",
    "                                lambda x: 'residential' if x in(residential_list)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f24154ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_residential_list = ['public','commercial','industrial', 'school','church','office',\n",
    "                        'retail','hotel','warehouse','kindergarten','civic','hospital']\n",
    "Mecklenburg_baseline['building'] = Mecklenburg_baseline['building'].apply(\\\n",
    "                                lambda x: 'non_residential' if x in(non_residential_list)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b2d7937",
   "metadata": {},
   "outputs": [],
   "source": [
    "unknown_list = ['yes','detached','terrace', 'garage','roof','shed',\n",
    "                'parking','garages','greenhouse','static_caravan','service','construction',\n",
    "                'misc_buildings']\n",
    "Mecklenburg_baseline['building'] = Mecklenburg_baseline['building'].apply(\\\n",
    "                                lambda x: 'unknown' if x in(unknown_list)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db8a56c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48036605",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline.groupby('building').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8757a526",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline.groupby(['official_type','building']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79e5d31e",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline.to_file(\"./Mecklenburg_baseline/Mecklenburg_baseline.shp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98b96140",
   "metadata": {},
   "outputs": [],
   "source": [
    "Mecklenburg_baseline.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1349e828",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8bf29c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nominal_transform(input_data, value_tags):\n",
    "    ohe = preprocessing.OneHotEncoder(sparse = False)\n",
    "    for tag in value_tags:\n",
    "        ohe_results = ohe.fit_transform(input_data[[tag]])\n",
    "        input_data.reset_index(inplace=True, drop=True)\n",
    "        input_data = pd.concat([input_data, \\\n",
    "            pd.DataFrame(ohe_results, columns=ohe.categories_[0].tolist())], axis=1)\n",
    "        input_data.drop(columns=[tag], inplace=True)\n",
    "        \n",
    "    return input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "173346ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# nominal transform\n",
    "\n",
    "#value_tags = ['building']\n",
    "value_tags = ['building','landuse']\n",
    "nominal_data = input_data.copy()\n",
    "nominal_data = nominal_transform(nominal_data, value_tags)\n",
    "nominal_features = list(nominal_data.columns[2:])\n",
    "nominal_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c257271",
   "metadata": {},
   "outputs": [],
   "source": [
    "nominal_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f34f0362",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(nominal_data.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b2b4f09",
   "metadata": {},
   "outputs": [],
   "source": [
    "nominal_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71c1299e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# latest, building and landuse\n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "weights = {0:2.0, 1:1.0}\n",
    "\n",
    "# Split data into 80% train and 20% test subsets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "#dt_split = DecisionTreeClassifier(min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "# Learn on the train subset\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on the test subset\n",
    "predicted = dt_split.predict(X_test)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(y_test, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(y_test, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72b896e9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "091547ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# result for generating output shape file\n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "#weights = 'balanced'\n",
    "weights = {0:2.0, 1:1.0}\n",
    "dt = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "y_pred = cross_val_predict(dt, X, y, cv=10)\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(y, y_pred)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt}:\\n\"\n",
    "    f\"{metrics.classification_report(y, y_pred)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88118eb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data = nominal_data.copy()\n",
    "output_data.drop(columns=['official_type','buffer_cat2_30', 'buffer_cat2_60','buffer_cat2_90'], inplace=True)\n",
    "output_data.insert(1,'official',y)\n",
    "output_data.insert(2,'predicted',y_pred)\n",
    "output_filtered = output_data.copy()\n",
    "#output_filtered = output_filtered[output_filtered['official'] != output_filtered['predicted']]\n",
    "output_filtered['official'] = \\\n",
    "    output_filtered['official'].apply(lambda x: 'RES' if x == 1 else 'NON_RES')\n",
    "output_filtered['predicted'] = \\\n",
    "    output_filtered['predicted'].apply(lambda x: 'RES' if x == 1 else 'NON_RES')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e6e6065",
   "metadata": {},
   "outputs": [],
   "source": [
    "#output_filtered.to_file(\"./Datasets/OSM/Geopandas_Output/filtered_output.shp\")\n",
    "output_data.to_file(\"./Mecklenburg_Predictions/Mecklenburg_Predictions.shp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81e2631e",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84ad2181",
   "metadata": {},
   "outputs": [],
   "source": [
    "#X.to_pickle(\"./Mecklenburg_Processed_Data/Mecklenburg_X.pkl\")  \n",
    "#y.to_pickle(\"./Mecklenburg_Processed_Data/Mecklenburg_y.pkl\")\n",
    "nominal_data.to_pickle(\"./Mecklenburg_Processed_Data/Mecklenburg_nominal_data.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "632bc94f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "600a688c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test on FFX 20% \n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "\n",
    "FFX_nominal = pd.read_pickle(\"./FFX_Processed_Data/FFX_nominal_data.pkl\")\n",
    "\n",
    "FFX_X = FFX_nominal[nominal_features]\n",
    "FFX_y = FFX_nominal[\"official_type\"]\n",
    "\n",
    "# Split data into 80% train and 20% test subsets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "\n",
    "FFX_X_train, FFX_X_test, FFX_y_train, FFX_y_test = train_test_split(\\\n",
    "    FFX_X, FFX_y, test_size=0.2, shuffle=True)\n",
    "\n",
    "#weights = 'balanced'\n",
    "weights = {0:2.0, 1:1.0}\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "#dt_split = DecisionTreeClassifier(min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "# Learn on the train subset\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on the test subset\n",
    "predicted = dt_split.predict(FFX_X_test)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(FFX_y_test, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(FFX_y_test, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29a3dea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test on FFX 100%\n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "\n",
    "FFX_nominal = pd.read_pickle(\"./FFX_Processed_Data/FFX_nominal_data.pkl\")\n",
    "\n",
    "FFX_X = FFX_nominal[nominal_features]\n",
    "FFX_y = FFX_nominal[\"official_type\"]\n",
    "\n",
    "# Split data into 80% train and 20% test subsets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "\n",
    "#weights = 'balanced'\n",
    "weights = {0:2.0, 1:1.0}\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "#dt_split = DecisionTreeClassifier(min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "# Learn on the train subset\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on the test subset\n",
    "predicted = dt_split.predict(FFX_X)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(FFX_y, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(FFX_y, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5ec1d30",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test on Boulder 100% data\n",
    "\n",
    "Boulder_nominal = pd.read_pickle(\"./Boulder_Processed_Data/Boulder_nominal_data.pkl\")\n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "\n",
    "Boulder_X = Boulder_nominal[nominal_features]\n",
    "Boulder_y = Boulder_nominal[\"official_type\"]\n",
    "\n",
    "#weights = 'balanced'\n",
    "weights = {0:2.0, 1:1.0}\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "\n",
    "# Learn on FFX\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on Boulder\n",
    "predicted = dt_split.predict(Boulder_X)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(Boulder_y, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(Boulder_y, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a95e204",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a554d69e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ablation Study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02284ece",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2389e2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data = input_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23473c02",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data.drop(columns=['buffer_cat1_30', 'buffer_cat1_60',\\\n",
    "                        'buffer_cat1_90', 'buffer_cat2_30', 'buffer_cat2_60', 'buffer_cat2_90',\\\n",
    "                        'buffer_cat3_30', 'buffer_cat3_60', 'buffer_cat3_90', 'buffer_cat4_30',\\\n",
    "                        'buffer_cat4_60', 'buffer_cat4_90', 'shop', 'website', 'brand', \\\n",
    "                        'amenity', 'building:levels', 'name', 'addr:street', 'source',\\\n",
    "                        'landuse', 'area',\\\n",
    "                        'parking_cat1', 'parking_cat2', 'parking_cat3', 'parking_cat4',\\\n",
    "                        'parking_cat5', 'parking_cat6', 'parking_cat7', 'parking_cat8',\\\n",
    "                        'parking_cat9'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0314e950",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd20fed1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# nominal transform\n",
    "\n",
    "#value_tags = ['building','landuse']\n",
    "#nominal_data = input_data.copy()\n",
    "value_tags = ['building']\n",
    "ablation_data = nominal_transform(ablation_data, value_tags)\n",
    "ablation_features = list(ablation_data.columns[2:])\n",
    "ablation_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60e1f348",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09e29b35",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91362a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = ablation_data[ablation_features]\n",
    "y = ablation_data[\"official_type\"]\n",
    "weights = {0:2.0, 1:1.0}\n",
    "\n",
    "# Split data into 80% train and 20% test subsets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "#dt_split = DecisionTreeClassifier(min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "# Learn on the train subset\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on the test subset\n",
    "predicted = dt_split.predict(X_test)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(y_test, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(y_test, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c13502cb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cd25f01",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "6e01ae1a5b6cec6d00c8cdda64aa3f8496ad213319a5f14e20b079aa3c21f8e3"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}