address_geocoding_and_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import geopy\n",
    "from geopy.geocoders import Nominatim\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from scipy.spatial.distance import cdist\n",
    "import logging\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the logger\n",
    "logger = logging.getLogger(__name__)\n",
    "logger.setLevel(logging.INFO)\n",
    "formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')\n",
    "stream_handler = logging.StreamHandler()\n",
    "stream_handler.setFormatter(formatter)\n",
    "logger.addHandler(stream_handler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the dataset\n",
    "data = pd.read_csv('data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assign your column names here\n",
    "address_column = 'Address'\n",
    "city_column = 'City'\n",
    "state_column = 'State'\n",
    "zip_column = 'Zip5'\n",
    "country_column = 'Country'\n",
    "# if you already have latitude and longitude columns, assign them here\n",
    "latitude_column = 'Latitude'\n",
    "longitude_column = 'Longitude'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If there is not an address column that contains the address, city, state, and zip code, then create one\n",
    "if 'full_address' not in data.columns:\n",
    "    data['full_address'] = data[address_column] + ', ' + data[city_column] + ', ' + data[state_column] + ', ' + data[zip_column].astype(str)\n",
    "\n",
    "# If there is no country column, default to 'US'\n",
    "if country_column not in data.columns:\n",
    "    data['Country'] = 'US'\n",
    "else:\n",
    "    # if it is there, append to have the full address\n",
    "    data['full_address'] = data['full_address'] + ', ' + data[country_column]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check dataset again\n",
    "data.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the geocoder\n",
    "geocoder = Nominatim(user_agent=\"my_app\")\n",
    "\n",
    "def geocode_address(address):\n",
    "    retries = 3\n",
    "    for i in range(retries):\n",
    "        try:\n",
    "            logger.info(f\"Geocoding address: {address}\")\n",
    "            result = geocoder.geocode(address)\n",
    "            logger.info(f\"Geocoded address: {result}\")\n",
    "            return result\n",
    "        except (geopy.exc.GeocoderTimedOut, geopy.exc.GeocoderUnavailable):\n",
    "            if i == retries - 1:\n",
    "                logger.warning(f\"Failed to geocode address: {address}\")\n",
    "                return None\n",
    "            logger.warning(f\"Geocoding attempt {i+1} failed for address: {address}\")\n",
    "            time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only needs to run if there is no geocoding already. If there is, skip to the next cell\n",
    "\n",
    "if data[latitude_column].empty and data[longitude_column].empty:\n",
    "    # Geocode the addresses\n",
    "    data['geocoded_address'] = data['full_address'].apply(lambda address: geocode_address(address))\n",
    "\n",
    "    # Extract the latitude and longitude from the geocoded address\n",
    "    data['Latitude'] = data['geocoded_address'].apply(lambda address: address.latitude if address else -999)\n",
    "    data['Longitude'] = data['geocoded_address'].apply(lambda address: address.longitude if address else -999)\n",
    "else:\n",
    "    logger.info(\"Latitude and longitude columns already exist. Skipping geocoding step.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess the data\n",
    "scaler = MinMaxScaler()\n",
    "scaled_data = scaler.fit_transform(data[['Latitude', 'Longitude']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine the optimal number of clusters using elbow analysis\n",
    "sum_of_squared_distances = []\n",
    "for k in range(1, 10):\n",
    "    kmeans = KMeans(n_clusters=k)\n",
    "    kmeans.fit(scaled_data)\n",
    "    labels = kmeans.labels_\n",
    "    centroid_distances = cdist(scaled_data, kmeans.cluster_centers_)\n",
    "    sum_of_squared_distances.append(centroid_distances.mean())\n",
    "\n",
    "# Get the elbow point\n",
    "elbow_point = sum_of_squared_distances.index(min(sum_of_squared_distances)) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize the elbow point analysis\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(range(1, 10), sum_of_squared_distances, 'bx-')\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('Average distance to centroid')\n",
    "plt.title('Elbow Method For Optimal k')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manually set the elbow point if you want to increase the number of clusters\n",
    "elbow_point = 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform k-means clustering with the optimal number of clusters\n",
    "kmeans = KMeans(n_clusters=elbow_point)\n",
    "kmeans.fit(scaled_data)\n",
    "labels = kmeans.labels_\n",
    "\n",
    "# Add the cluster labels to the dataset\n",
    "data['cluster'] = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize the clusters using plotly\n",
    "import plotly.express as px\n",
    "fig = px.scatter_mapbox(data, lat=\"Latitude\", lon=\"Longitude\", color=\"cluster\", zoom=10, height=800)\n",
    "fig.update_layout(mapbox_style=\"open-street-map\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the dataset with the new columns\n",
    "data.to_csv('data_with_clusters.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}