Boulder_City_Extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67e59447",
   "metadata": {},
   "outputs": [],
   "source": [
    "import osmium as osm\n",
    "import pandas as pd\n",
    "import geopandas\n",
    "import os.path\n",
    "import shapely.wkb as wkblib\n",
    "wkbfab = osm.geom.WKBFactory()\n",
    "import numpy as np\n",
    "import shapely.geometry\n",
    "\n",
    "from sklearn import tree\n",
    "import graphviz\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier, export_graphviz\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn import preprocessing\n",
    "import jenkspy\n",
    "from sklearn.metrics import PrecisionRecallDisplay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5482a8c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AreaHandler(osm.SimpleHandler):\n",
    "    def __init__(self):\n",
    "        osm.SimpleHandler.__init__(self)\n",
    "        self.num_polys = 0\n",
    "        self.osm_data = []\n",
    "        self.roads_data = []\n",
    "        self.landuse_data = []\n",
    "        self.amenity_data = []\n",
    "        \n",
    "    def count_polygons(self):\n",
    "        self.num_polys += 1\n",
    "        \n",
    "    def way(self, w):\n",
    "        try:\n",
    "            wkb = wkbfab.create_linestring(w)\n",
    "            line = wkblib.loads(wkb, hex=True)\n",
    "            for tag in w.tags:\n",
    "                if (tag.k=='highway'):\n",
    "                    self.roads_data.append([w.id,\n",
    "                                       tag.v,\n",
    "                                       line])\n",
    "        except Exception:\n",
    "            pass    \n",
    "        \n",
    "    def area(self, a):\n",
    "        try:\n",
    "            wkb = wkbfab.create_multipolygon(a)\n",
    "            self.count_polygons()\n",
    "            # load into shapely\n",
    "            poly = wkblib.loads(wkb, hex=True)\n",
    "            for tag in a.tags:\n",
    "                if (tag.k=='building'):\n",
    "                    self.osm_data.append([a.id,                                   \n",
    "                                    tag.v,\n",
    "                                    list(poly.geoms)[0],\n",
    "                                    list(dict(a.tags).keys()),\n",
    "                                    dict(a.tags)])\n",
    "                if (tag.k=='landuse'):\n",
    "                    self.landuse_data.append([tag.v,\n",
    "                                    list(poly.geoms)[0],\n",
    "                                    ])\n",
    "                if (tag.k == 'amenity'):\n",
    "                    self.amenity_data.append([tag.v,\n",
    "                                    list(poly.geoms)[0],\n",
    "                                    ])\n",
    "        except Exception:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99e94997",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load official buildings\n",
    "buildings_path = \"./Datasets/Official/Boulder_admin_buildings/Boulder_admin_buildings.shp\"\n",
    "official_buildings = geopandas.read_file(buildings_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "591d70bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4edb997",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38ede48d",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings = official_buildings.rename(columns={'BLDGTYPE': 'official_type',})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20e1b99b",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3883b30a",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.groupby(['official_type']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6843b31",
   "metadata": {},
   "outputs": [],
   "source": [
    "#dissolved_buildings = official_buildings.dissolve()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07df5361",
   "metadata": {},
   "outputs": [],
   "source": [
    "#dissolved_buildings.bounds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68a9c9f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbe76be9",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9dca6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_list = ['Agricultural','Foundation/Ruin','Garage/Shed','Misc','Parking Structure','Tank']\n",
    "official_buildings.drop(official_buildings[(official_buildings['official_type'].isin(drop_list))].index,\\\n",
    "                        inplace = True)\n",
    "official_buildings.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "013747c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d376cf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24ac1837",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2db9d5b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings['official_type'] = official_buildings['official_type'].apply(\\\n",
    "                                lambda x: 'RES' if (x == 'Residential'\n",
    "                                                ) else 'NON_RES')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1928548e",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2469cca5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a0361f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "areahandler = AreaHandler()\n",
    "areahandler.apply_file(\"./Datasets/OSM/Boulder_City_Buildings.osm.pbf\", locations=True)\n",
    "print(\"Number of polygons: %d\" % areahandler.num_polys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b86c913",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(areahandler.osm_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bab5ccd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the list into a pandas DataFrame\n",
    "data_colnames = ['id', 'building', 'geometry', 'all_tag_keys', 'all_tags']\n",
    "df = pd.DataFrame(areahandler.osm_data, columns=data_colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b901865",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a69c3029",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf = geopandas.GeoDataFrame(df, crs=4326, geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa0dc193",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bd58e1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "831733d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.groupby('building').count().sort_values(by=['building'], inplace=False, ascending=True).tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1af7265a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d60d34c3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea68123a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the list into a pandas DataFrame\n",
    "land_colnames = ['landuse', 'geometry']\n",
    "land_df = pd.DataFrame(areahandler.landuse_data, columns=land_colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cb4e6b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "343e1999",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d027da2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_df.groupby('landuse').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40a5f940",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_res = ['commercial', 'retail','industrial','grass','plant_nursery','quarry','railway','government',\\\n",
    "            'institutional']\n",
    "land_df['landuse'] = land_df['landuse'].apply(\\\n",
    "                                lambda x: 'non_res' if x in non_res else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "193ceef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_df.groupby('landuse').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c537f702",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_gdf = geopandas.GeoDataFrame(land_df, crs=4326, geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90869f59",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee64f53a",
   "metadata": {},
   "outputs": [],
   "source": [
    "land_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c830abd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land = gdf.sjoin(land_gdf, how=\"left\", predicate='intersects')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac908859",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e51920e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.drop_duplicates(subset='id',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaea46b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f693c93e",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf02b276",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.rename(columns={'index_right': 'index_landuse'},inplace=True)\n",
    "buildings_land['landuse'] = \\\n",
    "            buildings_land['landuse'].apply(lambda x: 'no_landuse' if pd.isnull(x) else x)\n",
    "buildings_land.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62f56365",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land[buildings_land['landuse'] == 'no_landuse'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb50fff5",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(buildings_land.index_landuse.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7abcf39e",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc3147aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby('landuse').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb1c611c",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby(['landuse','building']).count().head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e380651f",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land['land_building'] = buildings_land.apply(lambda x: \\\n",
    "    x['landuse'] if x['building'] == 'yes' else x['building'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05681fc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby(['landuse','land_building']).count().head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ecb39f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby(['landuse','land_building']).count().tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3edc282",
   "metadata": {},
   "outputs": [],
   "source": [
    "def enrich_buildings(buildings_land,tags_list):\n",
    "    for tag_val in tags_list:\n",
    "        buildings_land['land_building'] = buildings_land.apply(lambda x: \\\n",
    "            tag_val if x['landuse'] == tag_val else x['land_building'], axis=1)\n",
    "\n",
    "    return buildings_land"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3274fb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_list = ['residential','commercial','religious','recreation_ground','cemetery',\\\n",
    "            'construction','farmland','farmyard','forest','military']\n",
    "buildings_land = enrich_buildings(buildings_land,tags_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bba7b0d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby(['landuse','land_building']).count().head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67ab0f81",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby(['landuse','land_building']).count().tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83a51cf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.groupby(['land_building']).count().head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc1d6460",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1ab27bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_land.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a6925da",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10e83674",
   "metadata": {},
   "outputs": [],
   "source": [
    "official_buildings.crs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f82204bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf = buildings_land.sjoin(official_buildings, how=\"inner\", predicate='intersects')\n",
    "buildings_gdf.rename(columns={'index_right': 'index_official'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "255a8adb",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "093b6774",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d5c422b",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf.drop_duplicates(subset='id',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0385fac8",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc4005db",
   "metadata": {},
   "outputs": [],
   "source": [
    "#buildings_gdf = official_buildings.sjoin(gdf, how=\"inner\", predicate='intersects')\n",
    "#buildings_gdf = official_buildings.sjoin(buildings_land, how=\"inner\", predicate='intersects')\n",
    "#buildings_gdf = official_buildings.sjoin(buildings_land_parking, how=\"inner\", predicate='intersects')\n",
    "#buildings_gdf.rename(columns={'index_right': 'index_buildings'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da97ff06",
   "metadata": {},
   "outputs": [],
   "source": [
    "#buildings_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddf4724c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#buildings_gdf.drop_duplicates(subset='OBJECTID',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9759894b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#buildings_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce6deb43",
   "metadata": {},
   "outputs": [],
   "source": [
    "#buildings_gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "538630d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#buildings_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4463239",
   "metadata": {},
   "outputs": [],
   "source": [
    "#buildings_gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd7bdc59",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf.groupby(['landuse','land_building','official_type',]).count().head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "597d4203",
   "metadata": {},
   "outputs": [],
   "source": [
    "buildings_gdf.groupby(['landuse','land_building','official_type',]).count().tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f40ea983",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lookup all rows of above buildings_gdf.groupby(['landuse','land_building','official_type',]).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebc8b78c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4b70dc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(areahandler.roads_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b57dba89",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the list of roads into a pandas DataFrame\n",
    "roads_colnames = ['id', 'highway', 'geometry']\n",
    "roads_df = pd.DataFrame(areahandler.roads_data, columns=roads_colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c139b63",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89ce21e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91296714",
   "metadata": {},
   "outputs": [],
   "source": [
    "road_categories = roads_df.groupby(['highway'])['id'].count().reset_index(name=\"count\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d51b7750",
   "metadata": {},
   "outputs": [],
   "source": [
    "road_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fc5712d",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1 = roads_df[(roads_df['highway'] == 'residential') | (roads_df['highway'] == 'living_street')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83d0adc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1 = roads_cat1.assign(highway='residential')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f3d92c",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85825be5",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1.groupby('highway').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56b301fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1 = geopandas.GeoDataFrame(roads_cat1,crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5530b297",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_list = ['primary', 'secondary','tertiary']\n",
    "roads_cat2 = roads_df[roads_df['highway'].isin(roads_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a926940e",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2 = roads_cat2.assign(highway='non_res_road')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62402081",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf52ab86",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2.groupby('highway').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "087e95f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2 = geopandas.GeoDataFrame(roads_cat2,crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a16c4430",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3 = roads_df[(roads_df['highway'] == 'motorway') | (roads_df['highway'] == 'trunk')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05f6c69b",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3 = roads_cat3.assign(highway='roads_cat3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb9f5c61",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68bb9f16",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3.groupby('highway').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0364e102",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3 = geopandas.GeoDataFrame(roads_cat3,crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4254b99",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4 = roads_df[(roads_df['highway'] == 'service')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "add4f0c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2f7ecb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4.groupby('highway').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd352251",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4 = geopandas.GeoDataFrame(roads_cat4,crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84e0e945",
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_buffers(radius, road_category):\n",
    "    road_category['geometry'] = road_category['geometry'].to_crs(epsg=32610).buffer(radius)\n",
    "    road_category.to_crs(epsg=4326, inplace=True)\n",
    "    road_category.rename(columns={'highway': 'buffered_highway'}, inplace=True)\n",
    "    road_category = road_category.dissolve()\n",
    "    \n",
    "    return road_category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "929ce362",
   "metadata": {},
   "outputs": [],
   "source": [
    "# approx in meters unit: 1 decimal degree equals 111.32 km \n",
    "#radius_list = [0.00027,0.00054,0.00081]\n",
    "radius_list = [30,60,90]                        # meters\n",
    "road_category = roads_cat1.copy()\n",
    "buffered_cat1_30 = apply_buffers(radius_list[0],road_category)\n",
    "buffered_cat1_60 = apply_buffers(radius_list[1],road_category)\n",
    "buffered_cat1_90 = apply_buffers(radius_list[2],road_category)\n",
    "\n",
    "road_category = roads_cat2.copy()\n",
    "buffered_cat2_30 = apply_buffers(radius_list[0],road_category)\n",
    "buffered_cat2_60 = apply_buffers(radius_list[1],road_category)\n",
    "buffered_cat2_90 = apply_buffers(radius_list[2],road_category)\n",
    "\n",
    "road_category = roads_cat3.copy()\n",
    "buffered_cat3_30 = apply_buffers(radius_list[0],road_category)\n",
    "buffered_cat3_60 = apply_buffers(radius_list[1],road_category)\n",
    "buffered_cat3_90 = apply_buffers(radius_list[2],road_category)\n",
    "\n",
    "road_category = roads_cat4.copy()\n",
    "buffered_cat4_30 = apply_buffers(radius_list[0],road_category)\n",
    "buffered_cat4_60 = apply_buffers(radius_list[1],road_category)\n",
    "buffered_cat4_90 = apply_buffers(radius_list[2],road_category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a49ef419",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat1.plot()\n",
    "buffered_cat1_30.plot()\n",
    "buffered_cat1_60.plot()\n",
    "buffered_cat1_90.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d476b714",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat2.plot()\n",
    "buffered_cat2_30.plot()\n",
    "buffered_cat2_60.plot()\n",
    "buffered_cat2_90.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9238492",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat3.plot()\n",
    "buffered_cat3_30.plot()\n",
    "buffered_cat3_60.plot()\n",
    "buffered_cat3_90.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08959842",
   "metadata": {},
   "outputs": [],
   "source": [
    "roads_cat4.plot()\n",
    "buffered_cat4_30.plot()\n",
    "buffered_cat4_60.plot()\n",
    "buffered_cat4_90.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29b9ddd0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e966c67",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34ecd949",
   "metadata": {},
   "outputs": [],
   "source": [
    "def join_buildings(buffer_category, roads_category):\n",
    "    buffered_category = {\n",
    "        'cat1_30': buffered_cat1_30,\n",
    "        'cat1_60': buffered_cat1_60,\n",
    "        'cat1_90': buffered_cat1_90,\n",
    "        'cat2_30': buffered_cat2_30,\n",
    "        'cat2_60': buffered_cat2_60,\n",
    "        'cat2_90': buffered_cat2_90,\n",
    "        'cat3_30': buffered_cat3_30,\n",
    "        'cat3_60': buffered_cat3_60,\n",
    "        'cat3_90': buffered_cat3_90,\n",
    "        'cat4_30': buffered_cat4_30,\n",
    "        'cat4_60': buffered_cat4_60,\n",
    "        'cat4_90': buffered_cat4_90,\n",
    "\n",
    "    }.get(buffer_category)\n",
    "\n",
    "    joined = buildings_gdf.sjoin(buffered_category, how='left')\n",
    "    #joined = buildings_gdf.sjoin(buffered_category, how=\"left\", predicate='intersects')\n",
    "    joined.rename(columns={'index_right': 'index_roads'}, inplace=True)\n",
    "    #joined.drop_duplicates(subset='id_left',inplace=True)\n",
    "    if roads_category:\n",
    "        joined['buffered_highway'] = \\\n",
    "            joined['buffered_highway'].apply(lambda x: 'not_residential' if pd.isnull(x) else x)\n",
    "    else:\n",
    "        joined['buffered_highway'] = \\\n",
    "            joined['buffered_highway'].apply(lambda x: 'residential' if pd.isnull(x) else x)\n",
    "\n",
    "    joined.reset_index(inplace=True, drop=True)\n",
    "    \n",
    "    return joined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a0609f0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51f75745",
   "metadata": {},
   "outputs": [],
   "source": [
    "buffer_categories = ['cat1_30', 'cat1_60', 'cat1_90','cat2_30','cat2_60','cat2_90',\\\n",
    "                    'cat3_30', 'cat3_60', 'cat3_90','cat4_30','cat4_60','cat4_90']\n",
    "roads_category = 1                      # 1 is residential, 0 is ['primary', 'secondary','tertiary']\n",
    "joined_buildings1 = join_buildings(buffer_categories[0],roads_category)\n",
    "joined_buildings2 = join_buildings(buffer_categories[1],roads_category)\n",
    "joined_buildings3 = join_buildings(buffer_categories[2],roads_category)\n",
    "roads_category = 0\n",
    "joined_buildings4 = join_buildings(buffer_categories[3],roads_category)\n",
    "joined_buildings5 = join_buildings(buffer_categories[4],roads_category)\n",
    "joined_buildings6 = join_buildings(buffer_categories[5],roads_category)\n",
    "joined_buildings7 = join_buildings(buffer_categories[6],roads_category)\n",
    "joined_buildings8 = join_buildings(buffer_categories[7],roads_category)\n",
    "joined_buildings9 = join_buildings(buffer_categories[8],roads_category)\n",
    "joined_buildings10 = join_buildings(buffer_categories[9],roads_category)\n",
    "joined_buildings11 = join_buildings(buffer_categories[10],roads_category)\n",
    "joined_buildings12 = join_buildings(buffer_categories[11],roads_category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b2e0b5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(joined_buildings1.shape)\n",
    "print(joined_buildings2.shape)\n",
    "print(joined_buildings3.shape)\n",
    "print(joined_buildings4.shape)\n",
    "print(joined_buildings5.shape)\n",
    "print(joined_buildings6.shape)\n",
    "print(joined_buildings7.shape)\n",
    "print(joined_buildings8.shape)\n",
    "print(joined_buildings9.shape)\n",
    "print(joined_buildings10.shape)\n",
    "print(joined_buildings11.shape)\n",
    "print(joined_buildings12.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c3655dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings1.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "478458f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings1.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db2c7641",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings1[(joined_buildings1['land_building'] == 'commercial')\\\n",
    "               &(joined_buildings1['buffered_highway'] == 'residential')].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce21550c",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings2.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df180915",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings3.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "730ff030",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings4.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "065fab87",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings5.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98041d65",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings6.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7269c5bb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fc3422b",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings7.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aad3c58a",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings8.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "deecf989",
   "metadata": {},
   "outputs": [],
   "source": [
    "joined_buildings9.groupby(['official_type','buffered_highway']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aee2ab12",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6310c4d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(areahandler.osm_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de947b83",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_list = []\n",
    "for sublist in areahandler.osm_data:\n",
    "    keys_list.append(sublist[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "445f784f",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_list_flat = [val for sublist in keys_list for val in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "425c3ba3",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_keys = sorted(list(set(keys_list_flat)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f52bf4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e7fe704",
   "metadata": {},
   "outputs": [],
   "source": [
    "key_freq = []\n",
    "for key in unique_keys:\n",
    "    key_freq.append(keys_list_flat.count(key))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ed3c2f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(key_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d11ad262",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_tuples = list(zip(unique_keys, key_freq))\n",
    "keys_freq_df = pd.DataFrame(keys_tuples, columns = ['Key', 'Freq'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "489b0dfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.sort_values(by=['Freq'], inplace=True, ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b2f24ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8fedc34",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5dc6fd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.sort_values(by=['Key'], inplace=False, ascending=True).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca41dc73",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_freq_df.head(21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cd0ad78",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7fd7083",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_tags = ['source','addr:street','name','building:levels',\\\n",
    "                'roof:shape','amenity','brand','website','shop']\n",
    "num_keys = len(selected_tags)+2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71d0a6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(selected_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea139454",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40c9e15e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(input_data, value_tags, selected_tags):\n",
    "    col_pos = 2\n",
    "    for tag_key in selected_tags:\n",
    "        col_name = tag_key\n",
    "        input_data.insert(col_pos,col_name,0)\n",
    "        input_data[col_name] = input_data.apply(lambda x: 1 if tag_key in list(x['all_tag_keys']) \\\n",
    "            else 0, axis=1)\n",
    "\n",
    "    for tag_key in value_tags:\n",
    "        input_data[tag_key] = input_data.apply(lambda x: x['all_tags'].get(tag_key) \\\n",
    "            if tag_key in list(x['all_tag_keys']) else 0, axis=1)\n",
    "\n",
    "    # fix anomalies in FFX County data\n",
    "    input_data.drop(input_data[(input_data['building:levels'] == 'K-6')].index, inplace=True)\n",
    "    input_data.loc[(input_data['building:levels'] == '-1,1,2,3'),'building:levels']=4\n",
    "\n",
    "    input_data.drop(columns=['all_tag_keys', 'all_tags'], inplace=True)\n",
    "\n",
    "    input_data['official_type'] = \\\n",
    "    input_data['official_type'].apply(lambda x: 1 if x == 'RES' else 0)\n",
    "\n",
    "    return input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64f6052e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_buffers(input_data,col_names, buffers_list):\n",
    "    for num in range(0, len(col_names)):\n",
    "        input_data.insert(num+2,col_names[num],buffers_list[num])\n",
    "        input_data[col_names[num]] = input_data[col_names[num]].apply(lambda x: 1 if x == 'residential' else 0)\n",
    "\n",
    "    return input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbde3091",
   "metadata": {},
   "outputs": [],
   "source": [
    "#num_buffers = 6\n",
    "#buffer_loc = 0\n",
    "#value_tags = ['roof:shape','building:levels']\n",
    "#col_names = ['buffer_cat1_30','buffer_cat1_60','buffer_cat1_90',\\\n",
    "#            'buffer_cat2_30','buffer_cat2_60','buffer_cat2_90']\n",
    "#buffers_list = [joined_buildings1['buffered_highway'],joined_buildings2['buffered_highway'],\\\n",
    "#                joined_buildings3['buffered_highway'],joined_buildings4['buffered_highway'],\\\n",
    "#                joined_buildings5['buffered_highway'],joined_buildings6['buffered_highway']]\n",
    "\n",
    "num_buffers = 12\n",
    "buffer_loc = 0\n",
    "value_tags = ['roof:shape','building:levels']\n",
    "col_names = ['buffer_cat1_30','buffer_cat1_60','buffer_cat1_90',\\\n",
    "            'buffer_cat2_30','buffer_cat2_60','buffer_cat2_90',\\\n",
    "            'buffer_cat3_30','buffer_cat3_60','buffer_cat3_90',\\\n",
    "            'buffer_cat4_30','buffer_cat4_60','buffer_cat4_90']\n",
    "buffers_list = [joined_buildings1['buffered_highway'],joined_buildings2['buffered_highway'],\\\n",
    "                joined_buildings3['buffered_highway'],joined_buildings4['buffered_highway'],\\\n",
    "                joined_buildings5['buffered_highway'],joined_buildings6['buffered_highway'],\\\n",
    "                joined_buildings7['buffered_highway'],joined_buildings8['buffered_highway'],\\\n",
    "                joined_buildings9['buffered_highway'],joined_buildings10['buffered_highway'],\\\n",
    "                joined_buildings11['buffered_highway'],joined_buildings12['buffered_highway']]\n",
    "backup_data = joined_buildings1.copy()\n",
    "#input_data = input_data[['geometry','official_type','land_building','all_tag_keys','all_tags']]\n",
    "#input_data = input_data[['geometry','official_type','building','landuse','all_tag_keys','all_tags']]\n",
    "#input_data = input_data[['geometry','official_type','building','landuse',\\\n",
    "#                        'all_tag_keys','all_tags','id_left']]\n",
    "#input_data = input_data[['geometry','official_type','building','landuse','amenity_key','parking_area',\\\n",
    "#                        'all_tag_keys','all_tags']]\n",
    "backup_data = backup_data[['geometry','official_type','building','landuse','land_building',\\\n",
    "                        'all_tag_keys','all_tags','id_left']]\n",
    "backup_data.insert(len(backup_data.columns), 'area',backup_data['geometry'].to_crs(epsg=32610).area)\n",
    "backup_data = preprocess_data(backup_data,value_tags,selected_tags)\n",
    "\n",
    "backup_data = add_buffers(backup_data,col_names[buffer_loc:buffer_loc+num_buffers],\\\n",
    "                         buffers_list[buffer_loc:buffer_loc+num_buffers])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45beccc6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "335289bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(areahandler.amenity_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0da9b203",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the list of roads into a pandas DataFrame\n",
    "amenity_colnames = ['amenity_key', 'geometry']\n",
    "amenity_df = pd.DataFrame(areahandler.amenity_data, columns=amenity_colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb9c5e06",
   "metadata": {},
   "outputs": [],
   "source": [
    "amenity_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ed0e521",
   "metadata": {},
   "outputs": [],
   "source": [
    "amenity_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed7b14ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "amenity_categories = amenity_df.groupby(['amenity_key'])['geometry'].count().reset_index(name=\"count\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73269c82",
   "metadata": {},
   "outputs": [],
   "source": [
    "amenity_categories.tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9785f94",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_df = amenity_df[(amenity_df['amenity_key'] == 'parking') | (amenity_df['amenity_key'] == 'parking_space')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c826a14f",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_df = parking_df.assign(amenity_key='parking')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d57ce083",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23d0198f",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "752fb249",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf = geopandas.GeoDataFrame(parking_df.copy(),crs=\"EPSG:4326\",geometry='geometry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b492bc0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.insert(len(parking_gdf.columns), 'parking_area',parking_gdf['geometry'].to_crs(epsg=32610).area)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16e4c539",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c01d4955",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "feb830f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "hist = parking_gdf.hist(bins=100, grid=False, figsize=(12,8), color='#86bf91', zorder=2, rwidth=0.9)\n",
    "plt.yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b8d0fdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "breaks = jenkspy.jenks_breaks(parking_gdf['parking_area'], nb_class=3)\n",
    "print(breaks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efce7a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins_list = ['bin1','bin2','bin3']\n",
    "parking_gdf['cut_jenks'] = pd.cut(parking_gdf['parking_area'],\n",
    "                                bins=breaks,\n",
    "                                labels=bins_list,\n",
    "                                include_lowest=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6d60c49",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4650f7d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "for bin in bins_list:\n",
    "    parking_gdf.insert(len(parking_gdf.columns),bin,0)\n",
    "    parking_gdf[bin] = parking_gdf.apply(lambda x: 1 if x['cut_jenks'] == bin\\\n",
    "                                     else 0, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "264a9047",
   "metadata": {},
   "outputs": [],
   "source": [
    "parking_gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73bb2f0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1a993e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "radius_list = [30,60,90]\n",
    "counter = 1\n",
    "for bin in bins_list:\n",
    "    for radius in radius_list:\n",
    "        parking_buffer = parking_gdf.copy()\n",
    "        parking_buffer = parking_buffer[['geometry',bin]]\n",
    "        col_name = 'parking_cat'+str(counter)\n",
    "        counter += 1\n",
    "        parking_buffer.rename(columns={bin: col_name}, inplace=True)\n",
    "        parking_buffer['geometry'] = parking_buffer['geometry'].to_crs(epsg=32610).buffer(radius)\n",
    "        parking_buffer.to_crs(epsg=4326, inplace=True)\n",
    "        buildings_parking = backup_data.sjoin(parking_buffer, how=\"left\", predicate='intersects')\n",
    "        buildings_parking[col_name] = buildings_parking[col_name].fillna(0)\n",
    "        buildings_parking.drop_duplicates(subset='id_left',inplace=True)\n",
    "        buildings_parking.reset_index(inplace=True, drop=True)\n",
    "        backup_data.insert(len(backup_data.columns),col_name,list(buildings_parking[col_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65e97f6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.drop(columns=['id_left'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f165b58b",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7559fb24",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f84c5abb",
   "metadata": {},
   "outputs": [],
   "source": [
    "backup_data.loc[(backup_data['building:levels'] == '1;3;2'),'building:levels']=3\n",
    "backup_data.loc[(backup_data['building:levels'] == '4 floors'),'building:levels']=4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b72476bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#backup_data.to_pickle(\"./Boulder_Backup_Data/Boulder.pkl\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cabd7d4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data = backup_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0ab28b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a3b2d56",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.drop(columns=['land_building', 'roof:shape'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2356e18",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7758f1f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('landuse').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5593f15f",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['landuse'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ef89d72",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2fac583",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('building').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69c4d811",
   "metadata": {},
   "outputs": [],
   "source": [
    "#input_data.to_pickle(\"./Boulder_Input_Data/Boulder.pkl\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f34fb47",
   "metadata": {},
   "outputs": [],
   "source": [
    "#type(input_data['building'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e25b2c30",
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_array = np.array(['a','b','c','dormitory', 'office', 'yes', 'university'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f53145cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "277f12e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#a = np.array([1, 2, 3, 2, 4, 1,9])\n",
    "#b = np.array([3, 4, 5, 6,7,8])\n",
    "#np.setxor1d(a, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "350869de",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['building'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dba6cc25",
   "metadata": {},
   "outputs": [],
   "source": [
    "common_features = np.array(['apartments', 'church', 'civic', 'commercial', 'construction',\n",
    "       'detached', 'dormitory', 'garage', 'garages', 'greenhouse',\n",
    "       'hospital', 'hotel', 'house', 'industrial', 'kindergarten',\n",
    "       'office', 'parking', 'public', 'residential', 'retail', 'roof',\n",
    "       'school', 'semidetached_house', 'service', 'shed',\n",
    "       'static_caravan', 'terrace', 'warehouse', 'yes'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "910fccf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_buildings = np.setxor1d(input_data['building'].unique(), common_features).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46f87c2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_buildings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "629c0fbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(unique_buildings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76f4039a",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data[input_data['building'].isin(unique_buildings)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6610d7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['building'] = input_data['building'].apply(\\\n",
    "                                lambda x: 'misc_buildings' if x in(unique_buildings)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5910150",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data[input_data['building'].isin(unique_buildings)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a9029e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bafddb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('building').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf5c1fd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "common_landuse = np.array(['non_res', 'no_landuse', 'residential', 'construction',\n",
    "       'forest', 'recreation_ground', 'cemetery', 'farmyard'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ec3e842",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_landuse = np.setxor1d(input_data['landuse'].unique(), common_landuse).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96009d26",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_landuse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c468c9cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data[input_data['landuse'].isin(unique_landuse)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "288ca632",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['landuse'] = input_data['landuse'].apply(\\\n",
    "                                lambda x: 'misc_landuse' if x in(unique_landuse)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5657742d",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data[input_data['landuse'].isin(unique_landuse)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b38adcae",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('landuse').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea4f43b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data['landuse'].replace(['residential','non_res','forest','cemetery','recreation_ground',\\\n",
    "                                'construction','farmyard'],['residential_landuse','non_res_landuse',\\\n",
    "                                'forest_landuse','cemetery_landuse','recreation_ground_landuse',\\\n",
    "                                'construction_landuse','farmyard_landuse'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23cdf7bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.groupby('landuse').count().sort_values(by=['geometry'], inplace=False, ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9112a48",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7793b94c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6d4a27c",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline = input_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41ce8822",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "191b3f05",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline = Boulder_baseline[['geometry', 'official_type', 'building']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "288e868b",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c8efb25",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "735e3569",
   "metadata": {},
   "outputs": [],
   "source": [
    "residential_list = ['residential','apartments','dormitory', 'house','semidetached_house']\n",
    "Boulder_baseline['building'] = Boulder_baseline['building'].apply(\\\n",
    "                                lambda x: 'residential' if x in(residential_list)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f24154ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_residential_list = ['public','commercial','industrial', 'school','church','office',\n",
    "                        'retail','hotel','warehouse','kindergarten','civic','hospital']\n",
    "Boulder_baseline['building'] = Boulder_baseline['building'].apply(\\\n",
    "                                lambda x: 'non_residential' if x in(non_residential_list)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b2d7937",
   "metadata": {},
   "outputs": [],
   "source": [
    "unknown_list = ['yes','detached','terrace', 'garage','roof','shed',\n",
    "                'parking','garages','greenhouse','static_caravan','service','construction',\n",
    "                'misc_buildings']\n",
    "Boulder_baseline['building'] = Boulder_baseline['building'].apply(\\\n",
    "                                lambda x: 'unknown' if x in(unknown_list)\n",
    "                                                else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db8a56c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48036605",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline.groupby('building').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8757a526",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline.groupby(['official_type','building']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79e5d31e",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline.to_file(\"./Boulder_baseline/Boulder_baseline.shp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98b96140",
   "metadata": {},
   "outputs": [],
   "source": [
    "Boulder_baseline.groupby('official_type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87737ec9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8bf29c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nominal_transform(input_data, value_tags):\n",
    "    ohe = preprocessing.OneHotEncoder(sparse = False)\n",
    "    for tag in value_tags:\n",
    "        ohe_results = ohe.fit_transform(input_data[[tag]])\n",
    "        input_data.reset_index(inplace=True, drop=True)\n",
    "        input_data = pd.concat([input_data, \\\n",
    "            pd.DataFrame(ohe_results, columns=ohe.categories_[0].tolist())], axis=1)\n",
    "        input_data.drop(columns=[tag], inplace=True)\n",
    "        \n",
    "    return input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "173346ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# nominal transform\n",
    "\n",
    "#value_tags = ['roof:shape','building','landuse']\n",
    "#value_tags = ['roof:shape','land_building']\n",
    "#value_tags = ['roof:shape','building']\n",
    "value_tags = ['building','landuse']\n",
    "nominal_data = input_data.copy()\n",
    "#nominal_data['roof:shape'] = nominal_data['roof:shape'].astype(str)\n",
    "nominal_data = nominal_transform(nominal_data, value_tags)\n",
    "nominal_features = list(nominal_data.columns[2:])\n",
    "nominal_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c257271",
   "metadata": {},
   "outputs": [],
   "source": [
    "nominal_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f34f0362",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(nominal_data.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60b982f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# latest, building and landuse\n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "weights = {0:2.0, 1:1.0}\n",
    "\n",
    "# Split data into 80% train and 20% test subsets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "#dt_split = DecisionTreeClassifier(min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "# Learn on the train subset\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on the test subset\n",
    "predicted = dt_split.predict(X_test)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(y_test, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(y_test, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5131d6e3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53f47ace",
   "metadata": {},
   "outputs": [],
   "source": [
    "# result for generating output shape file\n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "#weights = 'balanced'\n",
    "weights = {0:2.0, 1:1.0}\n",
    "dt = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "y_pred = cross_val_predict(dt, X, y, cv=10)\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(y, y_pred)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt}:\\n\"\n",
    "    f\"{metrics.classification_report(y, y_pred)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88118eb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data = nominal_data.copy()\n",
    "output_data.drop(columns=['official_type','buffer_cat2_30', 'buffer_cat2_60','buffer_cat2_90'], inplace=True)\n",
    "output_data.insert(1,'official',y)\n",
    "output_data.insert(2,'predicted',y_pred)\n",
    "output_filtered = output_data.copy()\n",
    "#output_filtered = output_filtered[output_filtered['official'] != output_filtered['predicted']]\n",
    "output_filtered['official'] = \\\n",
    "    output_filtered['official'].apply(lambda x: 'RES' if x == 1 else 'NON_RES')\n",
    "output_filtered['predicted'] = \\\n",
    "    output_filtered['predicted'].apply(lambda x: 'RES' if x == 1 else 'NON_RES')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e6e6065",
   "metadata": {},
   "outputs": [],
   "source": [
    "#output_filtered.to_file(\"./Datasets/OSM/Geopandas_Output/filtered_output.shp\")\n",
    "output_data.to_file(\"./Boulder_Predictions/Boulder_Predictions.shp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c466fefa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29a3dea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test on FFX 100%\n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "\n",
    "FFX_nominal = pd.read_pickle(\"./FFX_Processed_Data/FFX_nominal_data.pkl\")\n",
    "\n",
    "FFX_X = FFX_nominal[nominal_features]\n",
    "FFX_y = FFX_nominal[\"official_type\"]\n",
    "\n",
    "# Split data into 80% train and 20% test subsets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "\n",
    "#weights = 'balanced'\n",
    "weights = {0:2.0, 1:1.0}\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "#dt_split = DecisionTreeClassifier(min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "# Learn on the train subset\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on the test subset\n",
    "predicted = dt_split.predict(FFX_X)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(FFX_y, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(FFX_y, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "444a5fe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test on Mecklenburg 100% data\n",
    "\n",
    "Mecklenburg_nominal = pd.read_pickle(\"./Mecklenburg_Processed_Data/Mecklenburg_nominal_data.pkl\")\n",
    "\n",
    "X = nominal_data[nominal_features]\n",
    "y = nominal_data[\"official_type\"]\n",
    "\n",
    "Mecklenburg_X = Mecklenburg_nominal[nominal_features]\n",
    "Mecklenburg_y = Mecklenburg_nominal[\"official_type\"]\n",
    "\n",
    "#weights = 'balanced'\n",
    "weights = {0:2.0, 1:1.0}\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "\n",
    "# Learn on FFX\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on Mecklenburg\n",
    "predicted = dt_split.predict(Mecklenburg_X)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(Mecklenburg_y, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(Mecklenburg_y, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "510d58f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#X.to_pickle(\"./Boulder_Processed_Data/Boulder_X.pkl\")  \n",
    "#y.to_pickle(\"./Boulder_Processed_Data/Boulder_y.pkl\")\n",
    "nominal_data.to_pickle(\"./Boulder_Processed_Data/Boulder_nominal_data.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1109576",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a554d69e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ablation Study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02284ece",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2389e2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data = input_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e210d5a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data.drop(columns=['buffer_cat1_30', 'buffer_cat1_60',\\\n",
    "                        'buffer_cat1_90', 'buffer_cat2_30', 'buffer_cat2_60', 'buffer_cat2_90',\\\n",
    "                        'buffer_cat3_30', 'buffer_cat3_60', 'buffer_cat3_90', 'buffer_cat4_30',\\\n",
    "                        'buffer_cat4_60', 'buffer_cat4_90',\\\n",
    "                        \\\n",
    "                        \\\n",
    "                        'parking_cat1', 'parking_cat2', 'parking_cat3', 'parking_cat4',\\\n",
    "                        'parking_cat5', 'parking_cat6', 'parking_cat7', 'parking_cat8',\\\n",
    "                        'parking_cat9'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0314e950",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd20fed1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# nominal transform\n",
    "\n",
    "value_tags = ['building','landuse']\n",
    "#nominal_data = input_data.copy()\n",
    "#value_tags = ['building']\n",
    "ablation_data = nominal_transform(ablation_data, value_tags)\n",
    "ablation_features = list(ablation_data.columns[2:])\n",
    "ablation_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54554f71",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09e29b35",
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91362a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = ablation_data[ablation_features]\n",
    "y = ablation_data[\"official_type\"]\n",
    "weights = {0:2.0, 1:1.0}\n",
    "\n",
    "# Split data into 80% train and 20% test subsets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "    X, y, test_size=0.2, shuffle=True)\n",
    "dt_split = DecisionTreeClassifier(class_weight=weights,min_samples_split=20, \\\n",
    "                                random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "#dt_split = DecisionTreeClassifier(min_samples_split=20, random_state=0,min_impurity_decrease = 0.0001)\n",
    "\n",
    "# Learn on the train subset\n",
    "dt_split.fit(X_train, y_train)\n",
    "\n",
    "# Predict the target on the test subset\n",
    "predicted = dt_split.predict(X_test)\n",
    "\n",
    "# 1 = RES; 0 = NON_RES\n",
    "\n",
    "disp = metrics.ConfusionMatrixDisplay.from_predictions(y_test, predicted)\n",
    "disp.figure_.suptitle(\"Confusion Matrix\")\n",
    "print(f\"Confusion matrix:\\n{disp.confusion_matrix}\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(\n",
    "    f\"Classification report for classifier {dt_split}:\\n\"\n",
    "    f\"{metrics.classification_report(y_test, predicted)}\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3eedc74",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bb652df",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "6e01ae1a5b6cec6d00c8cdda64aa3f8496ad213319a5f14e20b079aa3c21f8e3"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}