diff --git a/replacement_mode_modeling/01_extract_db_data.ipynb b/replacement_mode_modeling/01_extract_db_data.ipynb
new file mode 100644
index 00000000..eea9d642
--- /dev/null
+++ b/replacement_mode_modeling/01_extract_db_data.ipynb
@@ -0,0 +1,1699 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "38b147ff",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import ast\n",
+ "import sys\n",
+ "import pickle\n",
+ "import importlib\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "\n",
+ "from pandas.api.types import is_string_dtype\n",
+ "from pathlib import Path\n",
+ "from uuid import UUID\n",
+ "from collections import defaultdict\n",
+ "\n",
+ "pd.set_option(\"display.max_columns\", 100)\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e550aa2b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "INCLUDE_TEST_USERS = False"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "39306a1d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Add path to your emission server here. Uncommented because the notebooks are run in the server.\n",
+ "# If running locally, you need to point this to the e-mission server repo.\n",
+ "# emission_path = Path(os.getcwd()).parent.parent / 'my_emission_server' / 'e-mission-server'\n",
+ "# sys.path.append(str(emission_path))\n",
+ "\n",
+ "# # Also add the home (viz_scripts) to the path\n",
+ "sys.path.append('../viz_scripts')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "94f673d6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import emission.core.get_database as edb\n",
+ "import emission.storage.timeseries.abstract_timeseries as esta"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bbde79d1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "REPLACED_MODE_DICT = {\n",
+ " \"Stage_database\": {\n",
+ " 'no_trip': 'no_trip',\n",
+ " 'no_travel': 'no_trip',\n",
+ " 'Unknown': 'unknown',\n",
+ " 'unknown': 'unknown',\n",
+ " 'bus': 'transit',\n",
+ " 'drove_alone': 'car',\n",
+ " 'bike': 'p_micro',\n",
+ " 'shared_ride': 's_car',\n",
+ " 'walk': 'walk',\n",
+ " 'train': 'transit',\n",
+ " 'bikeshare': 's_micro',\n",
+ " 'not_a trip': 'no_trip',\n",
+ " 'pilot_ebike': 'p_micro',\n",
+ " 'electric_car': 'car',\n",
+ " 'taxi': 'ridehail',\n",
+ " 'not_a_trip': 'no_trip',\n",
+ " 'run': 'walk',\n",
+ " 'scootershare': 's_micro',\n",
+ " 'tramway': 'transit',\n",
+ " 'free_shuttle': 'transit',\n",
+ " 'e-bike': 'p_micro',\n",
+ " 'rental_car': 'car',\n",
+ " 'train_+ bus': 'transit',\n",
+ " 'skateboard': 'p_micro',\n",
+ " 'snowboarding': 'p_micro',\n",
+ " 'e_bike': 'p_micro',\n",
+ " 'golf_cart': 'unknown',\n",
+ " 'emergency_vehicle with others': 's_car',\n",
+ " 'call_friend': 's_car',\n",
+ " 'no_replacement': 'no_travel',\n",
+ " 'doing_nothing': 'no_trip',\n",
+ " 'na': 'no_trip',\n",
+ " 'ebike': 'p_micro',\n",
+ " 'hiking': 'walk',\n",
+ " 'n/a': 'no_trip',\n",
+ " 'testing': 'unknown',\n",
+ " 'home': 'no_trip',\n",
+ " 'must_walk 3-5 mi a day for back': 'walk',\n",
+ " 'family': 's_car',\n",
+ " 'car': 'car',\n",
+ " 'pilot_e-bike': 'p_micro',\n",
+ " 'pilot_bike': 'p_micro',\n",
+ " 'time_spent on the clock at amazon': 'no_trip',\n",
+ " 'working': 'no_trip',\n",
+ " 'walk_at work': 'walk',\n",
+ " 'sitting_on my butt doing nothing': 'no_trip',\n",
+ " 'nothing._delivered food for work': 'no_trip',\n",
+ " 'train,_bus and walk': 'transit',\n",
+ " 'work_vehicle': 'car',\n",
+ " 'friend_picked me up': 's_car',\n",
+ " 'ski': 'p_micro',\n",
+ " 'not_accurate': 'unknown',\n",
+ " 'stolen_ebike': 'p_micro'\n",
+ " },\n",
+ " \"openpath_prod_durham\": {\n",
+ " 'Unknown': 'unknown',\n",
+ " 'bike': 'p_micro',\n",
+ " 'shared_ride': 's_car',\n",
+ " 'drove_alone': 'car',\n",
+ " 'bus': 'transit',\n",
+ " 'no_travel': 'no_trip',\n",
+ " 'scootershare': 's_micro',\n",
+ " 'walk': 'walk',\n",
+ " 'taxi': 'ridehail',\n",
+ " 'e_car_drove_alone': 'car',\n",
+ " 'bikeshare': 's_micro',\n",
+ " 'ebike': 'p_micro',\n",
+ " 'train': 'transit',\n",
+ " 'e_car_shared_ride': 's_car'\n",
+ " },\n",
+ " \"openpath_prod_mm_masscec\": {\n",
+ " 'Unknown': 'unknown',\n",
+ " 'drove_alone': 'car',\n",
+ " 'walk': 'walk',\n",
+ " 'shared_ride': 's_car',\n",
+ " 'bike': 'p_micro',\n",
+ " 'bikeshare': 's_micro',\n",
+ " 'no_travel': 'no_trip',\n",
+ " 'taxi': 'ridehail',\n",
+ " 'bus': 'transit',\n",
+ " 'scootershare': 's_micro',\n",
+ " 'train': 'transit',\n",
+ " 'walking': 'walk',\n",
+ " 'e_car_drove_alone': 'car'\n",
+ " },\n",
+ " \"openpath_prod_ride2own\": {\n",
+ " 'Unknown': 'unknown',\n",
+ " 'drove_alone': 'car',\n",
+ " 'walk': 'walk',\n",
+ " 'shared_ride': 's_car',\n",
+ " 'bike': 'p_micro',\n",
+ " 'no_travel': 'no_trip',\n",
+ " 'taxi': 'ridehail',\n",
+ " 'bus': 'transit',\n",
+ " 'train': 'transit',\n",
+ " 'e_car_drove_alone': 'car',\n",
+ " 'e_car_shared_ride': 's_car'\n",
+ " },\n",
+ " \"openpath_prod_uprm_nicr\": {\n",
+ " 'Unknown': 'unknown',\n",
+ " 'walk': 'walk',\n",
+ " 'drove_alone': 'car'\n",
+ " }\n",
+ "}\n",
+ "\n",
+ "SURVEY_DATA_DICT = {\n",
+ " \"Stage_database\": {\n",
+ " \"Unique User ID (auto-filled, do not edit)\": \"user_id\",\n",
+ " \"In which year were you born?\": \"birth_year\",\n",
+ " \"What is your gender?\": \"gender\",\n",
+ " \"Do you have a valid driver's license?\": \"has_drivers_license\",\n",
+ " \"Are you a student?\": \"is_student\",\n",
+ " \"What is the highest grade or degree that you have completed?\": \"highest_education\",\n",
+ " \"Do you work for either pay or profit?\": \"is_paid\",\n",
+ " \"Do you have more than one job?\": \"has_multiple_jobs\",\n",
+ " \"Do you work full-time or part-time at your primary job?\": \"primary_job_type\",\n",
+ " \"Which best describes your primary job?\": \"primary_job_description\",\n",
+ " \"How did you usually get to your primary job last week? \": \"primary_job_commute_mode\",\n",
+ " \"Thinking about your daily commute to work last week, how many minutes did it usually take to get from home to the primary job/work place?\": \"primary_job_commute_time\",\n",
+ " \"At your primary job, do you have the ability to set or change your own start time?\": \"is_primary_job_flexible\",\n",
+ " \"Do you have the option of working from home or an alternate location instead of going into your primary work place?\": \"primary_job_can_wfh\",\n",
+ " \"How many days per week do you usually work from home or an alternate location?\": \"wfh_days\",\n",
+ " \"Do you own or rent your place of residence?\": \"residence_ownership_type\",\n",
+ " \"What is your home type?\": \"residence_type\",\n",
+ " \"Please identify which category represents your total household income, before taxes, for last year.\": \"income_category\",\n",
+ " \"Including yourself, how many people live in your home?\": \"n_residence_members\",\n",
+ " \"How many children under age 18 live in your home?\": \"n_residents_u18\",\n",
+ " \"Including yourself, how many people have a driver's license in your household?\": \"n_residents_with_license\",\n",
+ " \"How many motor vehicles are owned, leased, or available for regular use by the people who currently live in your household?\": \"n_motor_vehicles\",\n",
+ " \"If you were unable to use your household vehicle(s), which of the following options would be available to you to get you from place to place?\": \"available_modes\",\n",
+ " \"Do you have a medical condition that makes it difficult to travel outside of the home?\": \"has_medical_condition\",\n",
+ " \"How long have you had this condition?\": \"medical_condition_duration\"\n",
+ " },\n",
+ " # Retrieved from: e-mission-phone/survey-resources/data-xls/demo-survey-v1.xlsx\n",
+ " \"openpath_prod_durham\": {\n",
+ " \"At_your_primary_job_do_you_ha\": \"is_primary_job_flexible\",\n",
+ " \"Which_best_describes_your_prim\": \"primary_job_description\",\n",
+ " \"Do_you_work_full_time_or_part_\": \"primary_job_type\",\n",
+ " \"Do_you_have_the_option_of_work\": \"primary_job_can_wfh\",\n",
+ " \"Please_describe_your_primary_job\": \"primary_job_description_2\",\n",
+ " \"Do_you_have_more_than_one_job\": \"has_multiple_jobs\",\n",
+ " # Two columns: how many days/week do you work & what days of the week do you work. \n",
+ " # the latter has only 4 NA values, the former has 45 NA values.\n",
+ " \"What_days_of_the_week_do_you_t\": \"wfh_days\",\n",
+ " \"How_many_days_do_you_usually_w_001\": \"n_wfh_days\",\n",
+ " # All these are NAs.\n",
+ " \"Which_one_below_describe_you_b\": \"description\",\n",
+ " \"What_is_your_race_ethnicity\": \"race_or_ethnicity\",\n",
+ " \"Are_you_a_student\": \"is_student\",\n",
+ " \"What_is_the_highest_grade_or_d\": \"highest_education\",\n",
+ " \"do_you_consider_yourself_to_be\": \"is_transgender\",\n",
+ " \"What_is_your_gender\": \"gender\",\n",
+ " \"How_old_are_you\": \"age\",\n",
+ " \"Are_you_a_paid_worker\": \"is_paid\",\n",
+ " \"Do_you_have_a_driver_license\": \"has_drivers_license\",\n",
+ " \"How_long_you_had_this_conditio\": \"medical_condition_duration\",\n",
+ " \"Including_yourself_how_many_w_001\": \"n_residents_u18\",\n",
+ " \"Including_yourself_how_many_p\": \"n_residence_members\",\n",
+ " \"Do_you_own_or_rent_your_home\": \"residence_ownership_type\",\n",
+ " \"Please_identify_which_category\": \"income_category\",\n",
+ " \"If_you_were_unable_to_use_your\": \"available_modes\",\n",
+ " \"Including_yourself_how_many_p_001\": \"n_residents_with_license\",\n",
+ " \"Including_yourself_how_many_w\": \"n_working_residents\",\n",
+ " \"What_is_your_home_type\": \"residence_type\",\n",
+ " \"How_many_motor_vehicles_are_ow\": \"n_motor_vehicles\",\n",
+ " \"Do_you_have_a_condition_or_han\": \"has_medical_condition\"\n",
+ " },\n",
+ " \"openpath_prod_mm_masscec\": {\n",
+ " # Same questions as Durham.\n",
+ " \"At_your_primary_job_do_you_ha\": \"is_primary_job_flexible\",\n",
+ " \"Which_best_describes_your_prim\": \"primary_job_description\",\n",
+ " \"Do_you_work_full_time_or_part_\": \"primary_job_type\",\n",
+ " \"Do_you_have_the_option_of_work\": \"primary_job_can_wfh\",\n",
+ " \"Please_describe_your_primary_job\": \"primary_job_description_2\",\n",
+ " \"Do_you_have_more_than_one_job\": \"has_multiple_jobs\",\n",
+ " # Two columns: how many days/week do you work & what days of the week do you work. \n",
+ " # the latter has only 4 NA values, the former has 45 NA values.\n",
+ " \"What_days_of_the_week_do_you_t\": \"wfh_days\",\n",
+ " \"How_many_days_do_you_usually_w_001\": \"n_wfh_days\",\n",
+ " # All these are NAs.\n",
+ " \"Which_one_below_describe_you_b\": \"description\",\n",
+ " \"What_is_your_race_ethnicity\": \"race_or_ethnicity\",\n",
+ " \"Are_you_a_student\": \"is_student\",\n",
+ " \"What_is_the_highest_grade_or_d\": \"highest_education\",\n",
+ " \"do_you_consider_yourself_to_be\": \"is_transgender\",\n",
+ " \"What_is_your_gender\": \"gender\",\n",
+ " \"How_old_are_you\": \"age\",\n",
+ " \"Are_you_a_paid_worker\": \"is_paid\",\n",
+ " \"Do_you_have_a_driver_license\": \"has_drivers_license\",\n",
+ " \"How_long_you_had_this_conditio\": \"medical_condition_duration\",\n",
+ " \"Including_yourself_how_many_w_001\": \"n_residents_u18\",\n",
+ " \"Including_yourself_how_many_p\": \"n_residence_members\",\n",
+ " \"Do_you_own_or_rent_your_home\": \"residence_ownership_type\",\n",
+ " \"Please_identify_which_category\": \"income_category\",\n",
+ " \"If_you_were_unable_to_use_your\": \"available_modes\",\n",
+ " \"Including_yourself_how_many_p_001\": \"n_residents_with_license\",\n",
+ " \"Including_yourself_how_many_w\": \"n_working_residents\",\n",
+ " \"What_is_your_home_type\": \"residence_type\",\n",
+ " \"How_many_motor_vehicles_are_ow\": \"n_motor_vehicles\",\n",
+ " \"Do_you_have_a_condition_or_han\": \"has_medical_condition\"\n",
+ " },\n",
+ " \"openpath_prod_ride2own\": {\n",
+ " # Same questions as Durham.\n",
+ " \"How_old_are_you\": \"age\",\n",
+ " \"What_is_your_gender\": \"gender\",\n",
+ " \"do_you_consider_yourself_to_be\": \"is_transgender\",\n",
+ " \"What_is_your_race_ethnicity\": \"race_or_ethnicity\",\n",
+ " \"Do_you_have_a_driver_license\": \"has_drivers_license\",\n",
+ " \"Are_you_a_student\": \"is_student\",\n",
+ " \"What_is_the_highest_grade_or_d\": \"highest_education\",\n",
+ " \"Are_you_a_paid_worker\": \"is_paid\",\n",
+ " \"Which_one_below_describe_you_b\": \"description\",\n",
+ " \"Do_you_own_or_rent_your_home\": \"residence_ownership_type\",\n",
+ " \"What_is_your_home_type\": \"residence_type\",\n",
+ " \"Please_identify_which_category\": \"income_category\",\n",
+ " \"Including_yourself_how_many_p\": \"n_residence_members\",\n",
+ " \"Including_yourself_how_many_w\": \"n_working_residents\",\n",
+ " \"Including_yourself_how_many_p_001\": \"n_residents_with_license\",\n",
+ " \"Including_yourself_how_many_w_001\": \"n_residents_u18\",\n",
+ " \"How_many_motor_vehicles_are_ow\": \"n_motor_vehicles\",\n",
+ " \"If_you_were_unable_to_use_your\": \"available_modes\",\n",
+ " \"Do_you_have_a_condition_or_han\": \"has_medical_condition\",\n",
+ " \"How_long_you_had_this_conditio\": \"medical_condition_duration\",\n",
+ " \"Do_you_have_more_than_one_job\": \"has_multiple_jobs\",\n",
+ " \"Do_you_work_full_time_or_part_\": \"primary_job_type\",\n",
+ " \"Which_best_describes_your_prim\": \"primary_job_description\",\n",
+ " \"Please_describe_your_primary_job\": \"primary_job_description_2\",\n",
+ " \"At_your_primary_job_do_you_ha\": \"is_primary_job_flexible\",\n",
+ " \"Do_you_have_the_option_of_work\": \"primary_job_can_wfh\",\n",
+ " \"How_many_days_do_you_usually_w_001\": \"n_wfh_days\",\n",
+ " \"What_days_of_the_week_do_you_t\": \"wfh_days\"\n",
+ " },\n",
+ " \"openpath_prod_uprm_nicr\": {\n",
+ " # Same as Durham!\n",
+ " \"At_your_primary_job_do_you_ha\": \"is_primary_job_flexible\",\n",
+ " \"Which_best_describes_your_prim\": \"primary_job_description\",\n",
+ " \"Do_you_work_full_time_or_part_\": \"primary_job_type\",\n",
+ " \"Do_you_have_the_option_of_work\": \"primary_job_can_wfh\",\n",
+ " \"Please_describe_your_primary_job\": \"primary_job_description_2\",\n",
+ " \"Do_you_have_more_than_one_job\": \"has_multiple_jobs\",\n",
+ " # Two columns: how many days/week do you work & what days of the week do you work. \n",
+ " # the latter has only 4 NA values, the former has 45 NA values.\n",
+ " \"What_days_of_the_week_do_you_t\": \"wfh_days\",\n",
+ " \"How_many_days_do_you_usually_w_001\": \"n_wfh_days\",\n",
+ " # All these are NAs.\n",
+ " \"Which_one_below_describe_you_b\": \"description\",\n",
+ " \"What_is_your_race_ethnicity\": \"race_or_ethnicity\",\n",
+ " \"Are_you_a_student\": \"is_student\",\n",
+ " \"What_is_the_highest_grade_or_d\": \"highest_education\",\n",
+ " \"do_you_consider_yourself_to_be\": \"is_transgender\",\n",
+ " \"What_is_your_gender\": \"gender\",\n",
+ " \"How_old_are_you\": \"age\",\n",
+ " \"Are_you_a_paid_worker\": \"is_paid\",\n",
+ " \"Do_you_have_a_driver_license\": \"has_drivers_license\",\n",
+ " \"How_long_you_had_this_conditio\": \"medical_condition_duration\",\n",
+ " \"Including_yourself_how_many_w_001\": \"n_residents_u18\",\n",
+ " \"Including_yourself_how_many_p\": \"n_residence_members\",\n",
+ " \"Do_you_own_or_rent_your_home\": \"residence_ownership_type\",\n",
+ " \"Please_identify_which_category\": \"income_category\",\n",
+ " \"If_you_were_unable_to_use_your\": \"available_modes\",\n",
+ " \"Including_yourself_how_many_p_001\": \"n_residents_with_license\",\n",
+ " \"Including_yourself_how_many_w\": \"n_working_residents\",\n",
+ " \"What_is_your_home_type\": \"residence_type\",\n",
+ " \"How_many_motor_vehicles_are_ow\": \"n_motor_vehicles\",\n",
+ " \"Do_you_have_a_condition_or_han\": \"has_medical_condition\"\n",
+ " }\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e171e277",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "DB_SOURCE = [\n",
+ " \"Stage_database\", # Does NOT have composite trips BUT has section modes and distances\n",
+ " \"openpath_prod_durham\", # Has composite trips\n",
+ " \"openpath_prod_mm_masscec\", # Has composite trips\n",
+ " \"openpath_prod_ride2own\", # Has composite trips\n",
+ " \"openpath_prod_uprm_nicr\" # Has composite trips\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "70fa3112",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "CURRENT_DB = DB_SOURCE[4]\n",
+ "\n",
+ "assert CURRENT_DB in DB_SOURCE"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "69008893",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "## Source: db_utils.py in op-admin-dashboard.\n",
+ "\n",
+ "BINARY_DEMOGRAPHICS_COLS = [\n",
+ " 'user_id',\n",
+ " '_id',\n",
+ "]\n",
+ "\n",
+ "EXCLUDED_DEMOGRAPHICS_COLS = [\n",
+ " 'data.xmlResponse', \n",
+ " 'data.name',\n",
+ " 'data.version',\n",
+ " 'data.label',\n",
+ " 'xmlns:jr',\n",
+ " 'xmlns:orx',\n",
+ " 'id',\n",
+ " 'start',\n",
+ " 'end',\n",
+ " 'attrxmlns:jr',\n",
+ " 'attrxmlns:orx',\n",
+ " 'attrid',\n",
+ " '__version__',\n",
+ " 'attrversion',\n",
+ " 'instanceID',\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "12cc0c54",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "## Source: scaffolding.py\n",
+ "\n",
+ "def expand_userinputs(labeled_ct):\n",
+ " '''\n",
+ " param: labeled_ct: a dataframe of confirmed trips, some of which have labels\n",
+ " params: labels_per_trip: the number of labels for each trip.\n",
+ " Currently, this is 2 for studies and 3 for programs, and should be \n",
+ " passed in by the notebook based on the input config.\n",
+ " If used with a trip-level survey, it could be even larger.\n",
+ " '''\n",
+ " # CASE 1 of https://github.com/e-mission/em-public-dashboard/issues/69#issuecomment-1256835867\n",
+ " if len(labeled_ct) == 0:\n",
+ " return labeled_ct\n",
+ " label_only = pd.DataFrame(labeled_ct.user_input.to_list(), index=labeled_ct.index)\n",
+ " # disp.display(label_only.head())\n",
+ " labels_per_trip = len(label_only.columns)\n",
+ " print(\"Found %s columns of length %d\" % (label_only.columns, labels_per_trip))\n",
+ " expanded_ct = pd.concat([labeled_ct, label_only], axis=1)\n",
+ " assert len(expanded_ct) == len(labeled_ct), \\\n",
+ " (\"Mismatch after expanding labels, expanded_ct.rows = %s != labeled_ct.rows %s\" %\n",
+ " (len(expanded_ct), len(labeled_ct)))\n",
+ " print(\"After expanding, columns went from %s -> %s\" %\n",
+ " (len(labeled_ct.columns), len(expanded_ct.columns)))\n",
+ " assert len(expanded_ct.columns) == len(labeled_ct.columns) + labels_per_trip, \\\n",
+ " (\"Mismatch after expanding labels, expanded_ct.columns = %s != labeled_ct.columns %s\" %\n",
+ " (len(expanded_ct.columns), len(labeled_ct.columns)))\n",
+ " # disp.display(expanded_ct.head())\n",
+ " return expanded_ct"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9a98e2fb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "## Source: scaffolding.py\n",
+ "\n",
+ "def data_quality_check(expanded_ct):\n",
+ " '''1. Delete rows where the mode_confirm was pilot_ebike and repalced_mode was pilot_ebike.\n",
+ " 2. Delete rows where the mode_confirm was pilot_ebike and repalced_mode was same_mode.\n",
+ " 3. Replace same_mode for the mode_confirm for Energy Impact Calcualtion.'''\n",
+ "\n",
+ " # TODO: This is only really required for the initial data collection around the minipilot\n",
+ " # in subsequent deployes, we removed \"same mode\" and \"pilot_ebike\" from the options, so the\n",
+ " # dataset did not contain of these data quality issues\n",
+ "\n",
+ " if 'replaced_mode' in expanded_ct.columns:\n",
+ " expanded_ct.drop(expanded_ct[(expanded_ct['mode_confirm'] == 'pilot_ebike') & (expanded_ct['replaced_mode'] == 'pilot_ebike')].index, inplace=True)\n",
+ " expanded_ct.drop(expanded_ct[(expanded_ct['mode_confirm'] == 'pilot_ebike') & (expanded_ct['replaced_mode'] == 'same_mode')].index, inplace=True)\n",
+ " expanded_ct['replaced_mode'] = np.where(expanded_ct['replaced_mode'] == 'same_mode',expanded_ct['mode_confirm'], expanded_ct['replaced_mode'])\n",
+ " \n",
+ " return expanded_ct"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fe37bf27",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "## Source: scaffolding.py\n",
+ "\n",
+ "uuid_df = pd.json_normalize(list(edb.get_uuid_db().find()))\n",
+ "\n",
+ "if not INCLUDE_TEST_USERS:\n",
+ " uuid_df = uuid_df.loc[~uuid_df.user_email.str.contains('_test_'), :]\n",
+ "\n",
+ "filtered = uuid_df.uuid.unique()\n",
+ "\n",
+ "agg = esta.TimeSeries.get_aggregate_time_series()\n",
+ "all_ct = agg.get_data_df(\"analysis/confirmed_trip\", None)\n",
+ "\n",
+ "print(f\"Before filtering, length={len(all_ct)}\")\n",
+ "participant_ct_df = all_ct.loc[all_ct.user_id.isin(filtered), :]\n",
+ "print(f\"After filtering, length={len(participant_ct_df)}\")\n",
+ "\n",
+ "expanded_ct = expand_userinputs(participant_ct_df)\n",
+ "expanded_ct = data_quality_check(expanded_ct)\n",
+ "print(expanded_ct.columns.tolist())\n",
+ "expanded_ct['replaced_mode'] = expanded_ct['replaced_mode'].fillna('Unknown')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "13536d14",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Additional preprocessing for replaced mode (if any)\n",
+ "\n",
+ "mode_counts = expanded_ct['replaced_mode'].value_counts()\n",
+ "drop_modes = mode_counts[mode_counts == 1].index.tolist()\n",
+ "\n",
+ "expanded_ct.drop(\n",
+ " index=expanded_ct.loc[expanded_ct.replaced_mode.isin(drop_modes)].index,\n",
+ " inplace=True\n",
+ ")\n",
+ "\n",
+ "# Additional modes to drop.\n",
+ "expanded_ct.drop(\n",
+ " index=expanded_ct.loc[expanded_ct.replaced_mode.isin(\n",
+ " # Remove all rows with air, boat, or weird answers.\n",
+ " ['houseboat', 'gondola', 'airline_flight', 'aircraft', 'zoo', 'air',\n",
+ " 'airplane', 'boat', 'flight', 'plane', 'meal', 'lunch']\n",
+ " )].index,\n",
+ " inplace=True\n",
+ ")\n",
+ "\n",
+ "expanded_ct.replaced_mode = expanded_ct.replaced_mode.apply(lambda x: REPLACED_MODE_DICT[CURRENT_DB][x])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "258844f4",
+ "metadata": {},
+ "source": [
+ "# Demographic pre-processing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7461a4d2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Demographics\n",
+ "\n",
+ "if CURRENT_DB != \"Stage_database\":\n",
+ "\n",
+ " decoded_uuids = [str(x) for x in filtered]\n",
+ "\n",
+ " ## Source: query_demographics() in op-admin-dashboard.\n",
+ " ts = esta.TimeSeries.get_aggregate_time_series()\n",
+ " entries = list(ts.find_entries([\"manual/demographic_survey\"]))\n",
+ "\n",
+ " available_key = {}\n",
+ " for entry in entries:\n",
+ " survey_key = list(entry['data']['jsonDocResponse'].keys())[0]\n",
+ " if survey_key not in available_key:\n",
+ " available_key[survey_key] = []\n",
+ "\n",
+ " # Minor modification: Added user_id check to filter users.\n",
+ " if str(entry['user_id']) in decoded_uuids:\n",
+ " available_key[survey_key].append(entry)\n",
+ "\n",
+ " dataframes = {}\n",
+ " for key, json_object in available_key.items():\n",
+ " df = pd.json_normalize(json_object)\n",
+ " dataframes[key] = df\n",
+ "\n",
+ " for key, df in dataframes.items():\n",
+ " if not df.empty:\n",
+ " for col in BINARY_DEMOGRAPHICS_COLS:\n",
+ " if col in df.columns:\n",
+ " df[col] = df[col].apply(str) \n",
+ " columns_to_drop = [col for col in df.columns if col.startswith(\"metadata\")]\n",
+ " df.drop(columns= columns_to_drop, inplace=True) \n",
+ " df.columns=[col.rsplit('.',1)[-1] if col.startswith('data.jsonDocResponse.') else col for col in df.columns]\n",
+ " for col in EXCLUDED_DEMOGRAPHICS_COLS:\n",
+ " if col in df.columns:\n",
+ " df.drop(columns= [col], inplace=True)\n",
+ "\n",
+ " survey_data = pd.DataFrame() \n",
+ " for v in dataframes.values():\n",
+ " survey_data = pd.concat([survey_data, v], axis=0, ignore_index=True)\n",
+ "else:\n",
+ " # Read the demographics.\n",
+ " # Ensure that you have access to this survey file and that it is placed in the given destination.\n",
+ " survey_data = pd.read_csv('../viz_scripts/Can Do Colorado eBike Program - en.csv')\n",
+ " survey_data.rename(columns={'Unique User ID (auto-filled, do not edit)': 'user_id'}, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "07922a00",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_section_durations(confirmed_trips: pd.DataFrame):\n",
+ " \n",
+ " import pandarallel\n",
+ "\n",
+ " # Initialize the parallel processing.\n",
+ " pandarallel.initialize(progress_bar=False)\n",
+ "\n",
+ " \"\"\"\n",
+ " Extract section-wise durations from trips for every trips.\n",
+ " \"\"\"\n",
+ "\n",
+ " # the inner function has access to these variables.\n",
+ " primary_key = 'analysis/inferred_section'\n",
+ " fallback_key = 'analysis/cleaned_section'\n",
+ "\n",
+ " def get_durations(user_id, trip_id):\n",
+ "\n",
+ " inferred_sections = esdt.get_sections_for_trip(key = primary_key,\n",
+ " user_id = user_id, trip_id = trip_id)\n",
+ "\n",
+ " if inferred_sections and len(inferred_sections) > 0:\n",
+ " return [x.data.duration for x in inferred_sections]\n",
+ " \n",
+ " print(\"Falling back to confirmed trips...\")\n",
+ "\n",
+ " cleaned_sections = esdt.get_sections_for_trip(key = fallback_key,\n",
+ " user_id = user_id, trip_id = trip_id)\n",
+ " \n",
+ " if cleaned_sections and len(cleaned_sections) > 0:\n",
+ " return [x.data.duration for x in cleaned_sections]\n",
+ "\n",
+ " return []\n",
+ "\n",
+ " confirmed_trips['section_durations'] = confirmed_trips.parallel_apply(\n",
+ " lambda x: get_durations(x.user_id, x.cleaned_trip), axis=1\n",
+ " )\n",
+ "\n",
+ " return confirmed_trips"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fe5a9dff",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if CURRENT_DB == \"Stage_database\":\n",
+ " \n",
+ " if os.path.exists('./data/cached_allceo_data.csv'):\n",
+ " \n",
+ " # Replace current instance of dataframe with the cached dataframe.\n",
+ " expanded_ct = pd.read_csv('./data/cached_allceo_data.csv')\n",
+ " expanded_ct.loc[expanded_ct.replaced_mode == 'no_travel', 'replaced_mode'] = 'no_trip'\n",
+ " else:\n",
+ " ## NOTE: Run this cell only if the cached CSV is not already available. It will take a LOT of time.\n",
+ " ## Benchmark timing: ~12 hours on a MacBook Pro (2017 model) with pandarallel, 4 workers.\n",
+ " expanded_ct = get_section_durations(expanded_ct)\n",
+ " expanded_ct.to_csv('./data/cached_allceo_data.csv', index=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a6be751e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(len(survey_data.user_id.unique()), len(expanded_ct.user_id.unique()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9ebc87d8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "survey_data.rename(SURVEY_DATA_DICT[CURRENT_DB], axis='columns', inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "522b1362",
+ "metadata": {},
+ "source": [
+ "### Demographic data preprocessing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "29bc7996",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# gtg\n",
+ "survey_data['ft_job'] = survey_data.primary_job_type.apply(\n",
+ " lambda x: 1 if str(x).lower() == 'full_time' else 0\n",
+ ")\n",
+ "survey_data.loc[~survey_data.ft_job.isin([0, 1]), 'ft_job'] = 0\n",
+ "\n",
+ "# gtg\n",
+ "survey_data['multiple_jobs'] = survey_data.has_multiple_jobs.apply(\n",
+ " lambda x: 1 if str(x).lower() == 'yes' else 0\n",
+ ")\n",
+ "survey_data.loc[~survey_data.multiple_jobs.isin([0, 1]), 'multiple_jobs'] = 0\n",
+ "\n",
+ "# gtg\n",
+ "survey_data.loc[\n",
+ " survey_data.n_motor_vehicles.isin(\n",
+ " ['prefer_not_to_say', 'Prefer not to say / Prefiero no decir.']\n",
+ " ), 'n_motor_vehicles'] = 0\n",
+ "\n",
+ "survey_data.loc[survey_data.n_motor_vehicles.isin(\n",
+ " ['more_than_3', '4+', 'more_than_4', 'more_than_3']\n",
+ "), 'n_motor_vehicles'] = 4\n",
+ "survey_data.n_motor_vehicles = survey_data.n_motor_vehicles.astype(int)\n",
+ "\n",
+ "# gtg\n",
+ "survey_data.has_drivers_license = survey_data.has_drivers_license.apply(\n",
+ " lambda x: 1 if str(x).lower() == 'yes' else 0\n",
+ ")\n",
+ "survey_data.loc[~survey_data.has_drivers_license.isin([0, 1]), 'has_drivers_license'] = 0\n",
+ "\n",
+ "survey_data.loc[survey_data.n_residents_u18 == 'prefer_not_to_say', 'n_residents_u18'] = 0\n",
+ "survey_data.n_residents_u18 = survey_data.n_residents_u18.astype(int)\n",
+ "\n",
+ "survey_data.loc[survey_data.n_residence_members == 'prefer_not_to_say', 'n_residence_members'] = 0\n",
+ "survey_data.n_residence_members = survey_data.n_residence_members.astype(int)\n",
+ "\n",
+ "survey_data.loc[survey_data.n_residents_with_license == 'prefer_not_to_say'] = 0\n",
+ "survey_data.loc[survey_data.n_residents_with_license == 'more_than_4', 'n_residents_with_license'] = 4\n",
+ "survey_data.n_residents_with_license = survey_data.n_residents_with_license.astype(int)\n",
+ "\n",
+ "# Handle abnormal inputs.\n",
+ "survey_data = survey_data[\n",
+ " (survey_data.n_residence_members - survey_data.n_residents_with_license >= 0) &\n",
+ " (survey_data.n_residence_members - survey_data.n_residents_u18 >= 0)\n",
+ "].reset_index(drop=True)\n",
+ "\n",
+ "# gtg\n",
+ "if CURRENT_DB != \"Stage_database\":\n",
+ " survey_data.n_working_residents = survey_data.n_working_residents.apply(\n",
+ " lambda x: 0 if x == 'prefer_not_to_say' else int(x)\n",
+ " )\n",
+ "else:\n",
+ " survey_data['n_working_residents'] = survey_data['n_residence_members'] - survey_data['n_residents_u18']\n",
+ " \n",
+ "survey_data = survey_data[survey_data.n_working_residents >= 0].reset_index(drop=True)\n",
+ "\n",
+ "# gtg\n",
+ "survey_data.is_paid = survey_data.is_paid.apply(lambda x: 1 if x == 'Yes' else 0)\n",
+ "survey_data.loc[~survey_data.is_paid.isin([0, 1]), 'is_paid'] = 0\n",
+ "\n",
+ "# gtg\n",
+ "survey_data.has_medical_condition = survey_data.has_medical_condition.apply(\n",
+ " lambda x: 1 if str(x).lower() == 'yes' else 0\n",
+ ")\n",
+ "survey_data.loc[~survey_data.has_medical_condition.isin([0, 1]), 'has_medical_condition'] = 0\n",
+ "\n",
+ "\n",
+ "## gtg\n",
+ "survey_data.is_student.replace({\n",
+ " 'Not a student': 0, \n",
+ " 'Yes - Full Time College/University': 1,\n",
+ " 'Yes - Vocation/Technical/Trade School': 1,\n",
+ " 'Yes - K-12th Grade including GED': 1, \n",
+ " 'Work': 0, \n",
+ " 'No': 0,\n",
+ " 'Prefer not to say': 0,\n",
+ " 'Yes - Part-Time College/University': 1,\n",
+ " 'Taking prerequisites missing for grad program ': 1, \n",
+ " 'Graduate': 1,\n",
+ " 'Fire Fighter 2 Training': 0,\n",
+ " 'By hours ': 0,\n",
+ " 'Custodian': 0, \n",
+ " 'taking classes toward early childhood licensure': 0,\n",
+ " 'Work at csu': 0,\n",
+ " 'not_a_student': 0, \n",
+ " 'yes___vocation_technical_trade_school': 1,\n",
+ " 'yes___part_time_college_university': 1,\n",
+ " 'prefer_not_to_say': 0, \n",
+ " 'yes___k_12th_grade_including_ged': 1,\n",
+ " 'yes___full_time_college_university': 1\n",
+ "}, inplace=True)\n",
+ "\n",
+ "survey_data.loc[~survey_data.is_student.isin([0, 1]), 'is_student'] = 0"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "aeb85637",
+ "metadata": {},
+ "source": [
+ "### Additinal Demographic Data Preprocessing"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9c069bd2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if CURRENT_DB == \"Stage_database\":\n",
+ " age = survey_data.birth_year.apply(\n",
+ " lambda x: 2024 - int(x) if int(x) > 100 else int(x)\n",
+ " )\n",
+ " \n",
+ " upper = age - (age % 5)\n",
+ " lower = upper + 5\n",
+ " new_col = (upper + 1).astype(str) + '___' + lower.astype(str) + '_years_old'\n",
+ " survey_data['age'] = new_col\n",
+ " \n",
+ " survey_data.loc[survey_data.age.isin([\n",
+ " '66___70_years_old', '71___75_years_old', '76___80_years_old', '81___85_years_old',\n",
+ " '151___155_years_old', \n",
+ " ]), 'age'] = '__65_years_old'\n",
+ " \n",
+ " survey_data.drop(columns=['birth_year'], inplace=True)\n",
+ "\n",
+ "else:\n",
+ " survey_data = survey_data[survey_data.age != 0].reset_index(drop=True)\n",
+ "\n",
+ "if survey_data.columns.isin(['primary_job_commute_mode', 'primary_job_commute_time']).all():\n",
+ " survey_data.drop(columns=['primary_job_commute_mode', 'primary_job_commute_time'], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f094cadd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def normalize_job_descriptions(db_name, df):\n",
+ " if db_name != 'Stage_database':\n",
+ " PRIMARY_JOB_DESCRIPTION_DICT = {\n",
+ " \"sales_or_service\": \"Sales or service\",\n",
+ " \"other\": \"Other\",\n",
+ " \"\": \"Other\",\n",
+ " \"professional__managerial__or_technical\": \"Professional, Manegerial, or Technical\",\n",
+ " \"manufacturing__construction__maintenance\": \"Manufacturing, construction, maintenance, or farming\",\n",
+ " \"clerical_or_administrative_support\": \"Clerical or administrative support\",\n",
+ " \"prefer_not_to_say\": \"Prefer not to say\"\n",
+ " }\n",
+ " \n",
+ " df.primary_job_description = df.primary_job_description.apply(\n",
+ " lambda x: PRIMARY_JOB_DESCRIPTION_DICT[x]\n",
+ " )\n",
+ " else:\n",
+ " df.primary_job_description = df.primary_job_description.str.strip()\n",
+ "\n",
+ " # Normalize the job description. Inspired from the 'e-bike trips by occupation' \n",
+ " # plot in the CanBikeCo full pilot paper.\n",
+ " df.loc[\n",
+ " df.primary_job_description.isin([\n",
+ " 'Paraprofessional', 'Education', 'education/early childhood', 'Teacher',\n",
+ " 'Education non-profit manager', 'Scientific research', 'Research',\n",
+ " 'Preschool Tracher'\n",
+ " ]), 'primary_job_description'\n",
+ " ] = 'Education'\n",
+ "\n",
+ " df.loc[\n",
+ " df.primary_job_description.isin([\n",
+ " 'Custodian', 'Custodial', 'Csu custodian', 'Janitorial',\n",
+ " 'Custodial Maintanace'\n",
+ " ]), 'primary_job_description'\n",
+ " ] = 'Custodial'\n",
+ "\n",
+ " df.loc[\n",
+ " df.primary_job_description.isin([\n",
+ " 'Inbound cs', 'Accounting Technician', \n",
+ " 'Clerical'\n",
+ " ]), 'primary_job_description'\n",
+ " ] = 'Clerical or administrative support'\n",
+ "\n",
+ " df.loc[\n",
+ " df.primary_job_description.isin([\n",
+ " 'Restaurant manager', 'Transportaion Services',\n",
+ " ]), 'primary_job_description'\n",
+ " ] = 'Sales or service'\n",
+ "\n",
+ " df.loc[\n",
+ " df.primary_job_description.isin([\n",
+ " 'Pastry chef and line cook', 'Cook', 'Chef', 'Dining Services',\n",
+ " 'Food Service', 'Cooking', 'Residential Dining Services', 'Line Cook'\n",
+ " ]), 'primary_job_description'\n",
+ " ] = 'Food service'\n",
+ "\n",
+ " df.loc[\n",
+ " df.primary_job_description.isin([\n",
+ " 'CNA', 'Caregiver/ Qmap', 'Health care', 'Nurse',\n",
+ " 'Healthcare', 'Medical', 'Medical field',\n",
+ " 'Family support'\n",
+ " ]), 'primary_job_description'\n",
+ " ] = 'Medical/healthcare'\n",
+ "\n",
+ " df.loc[\n",
+ " df.primary_job_description.isin([\n",
+ " 'Amazon', 'Hockey rink', 'Caregiver', 'Security', 'Nonprofit social work',\n",
+ " 'Therapeutic', 'Driver'\n",
+ " ]), 'primary_job_description'\n",
+ " ] = 'Other'\n",
+ "\n",
+ " df.loc[\n",
+ " df.primary_job_description.isin([\n",
+ " 'Hospital laundry', 'Matreal handler', 'Maintenance',\n",
+ " 'Co op laundry'\n",
+ " ]), 'primary_job_description'\n",
+ " ] = 'Manufacturing, construction, maintenance, or farming'\n",
+ "\n",
+ " # All others in Other\n",
+ " df.loc[\n",
+ " (df.primary_job_description.isna()) | (~df.primary_job_description.isin(\n",
+ " ['Education', 'Custodial', 'Clerical or administrative support', 'Sales or service'\n",
+ " 'Food service', 'Medical/healthcare', 'Manufacturing, construction, maintenance, or farming',\n",
+ " 'Other'])), \n",
+ " 'primary_job_description'\n",
+ " ] = 'Other'\n",
+ "\n",
+ " return df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0bf37859",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "INCOME_DICT = {\n",
+ " 'Stage_database': {\n",
+ " 'Prefer not to say': 0,\n",
+ " 'Less than $24,999': 1,\n",
+ " '$25,000-$49,999': 2,\n",
+ " '$50,000-$99,999': 3,\n",
+ " '$100,000 -$149,999': 4,\n",
+ " '$150,000-$199,999': 5,\n",
+ " '$150,000': 5,\n",
+ " '$150,000-$199,999': 6,\n",
+ " '$200,000 or more': 7\n",
+ " },\n",
+ " 'Others': {\n",
+ " 'prefer_not_to_say': 0, \n",
+ " 'less_than__24_999': 1,\n",
+ " '_25_000_to__49_999': 2,\n",
+ " '_50_000_to__99_999': 3,\n",
+ " '_100_000_to__149_999': 4,\n",
+ " '_150_000_to__199_999': 5\n",
+ " }\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "42b3163a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "survey_data = normalize_job_descriptions(CURRENT_DB, survey_data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fe2b18b6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if CURRENT_DB == 'Stage_database':\n",
+ " survey_data.income_category = survey_data.income_category.apply(\n",
+ " lambda x: INCOME_DICT['Stage_database'][x]\n",
+ " )\n",
+ "else:\n",
+ " survey_data.income_category = survey_data.income_category.apply(\n",
+ " lambda x: INCOME_DICT['Others'][x]\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b36672b9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.preprocessing import OneHotEncoder\n",
+ "\n",
+ "def generate_ohe_features(df, feature_name):\n",
+ " ohe = OneHotEncoder()\n",
+ " ohe.fit(df[[feature_name]])\n",
+ " return pd.DataFrame(\n",
+ " ohe.transform(df[[feature_name]]).todense(), \n",
+ " columns=ohe.get_feature_names_out(),\n",
+ " index=df.index\n",
+ " ), ohe"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dc8d1846",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "survey_data.reset_index(drop=True, inplace=True)\n",
+ "\n",
+ "ohe_features = ['highest_education', 'primary_job_description', 'gender', 'age']\n",
+ "\n",
+ "for ohe in ohe_features:\n",
+ " df, _ = generate_ohe_features(survey_data, ohe)\n",
+ " survey_data = survey_data.merge(right=df, left_index=True, right_index=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d2d6f8c1",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "to_drop = [\n",
+ " 'Timestamp', 'gender', 'highest_education', 'primary_job_type', 'primary_job_description', \n",
+ " 'primary_job_commute_mode', 'primary_job_commute_time', 'is_primary_job_flexible', \n",
+ " 'primary_job_can_wfh', 'wfh_days', 'Which one below describe you best?', 'residence_ownership_type', \n",
+ " 'residence_type', 'medical_condition_duration', 'has_multiple_jobs', 'age', '_id', 'data.ts',\n",
+ " 'primary_job_description_2', 'wfh_days', 'n_wfh_days', 'description', 'race_or_ethnicity', \n",
+ " 'highest_education', 'is_transgender', 'medical_condition_duration'\n",
+ "]\n",
+ "\n",
+ "for column in to_drop:\n",
+ " if column in survey_data.columns:\n",
+ " survey_data.drop(columns=[column], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "65039f73",
+ "metadata": {},
+ "source": [
+ "## Merge sensed data and demographics"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c7eb2e09",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Additional preprocessing to filter unwanted users from sensed trips data.\n",
+ "expanded_ct['user_id_join'] = expanded_ct['user_id'].apply(lambda x: str(x).replace('-', ''))\n",
+ "survey_data['user_id_join'] = survey_data['user_id'].apply(lambda x: str(x).replace('-', ''))\n",
+ "\n",
+ "survey_data.rename(columns={'user_id': 'survey_user_id'}, inplace=True)\n",
+ "\n",
+ "common = set(expanded_ct.user_id_join.unique()).intersection(\n",
+ " set(survey_data.user_id_join.unique())\n",
+ ")\n",
+ "\n",
+ "filtered_trips = expanded_ct.loc[expanded_ct.user_id_join.isin(common), :].reset_index(drop=True)\n",
+ "filtered_survey = survey_data.loc[survey_data.user_id_join.isin(common), :].reset_index(drop=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "53927d5f",
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "# Just to double-check.\n",
+ "print(len(filtered_trips.user_id.unique()), len(filtered_survey.survey_user_id.unique()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "daed8fb0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Compute the section_*_argmax.\n",
+ "\n",
+ "def compute_argmax(db: str, row):\n",
+ " \n",
+ " if db != 'Stage_database':\n",
+ " \n",
+ " sections = row['inferred_section_summary']\n",
+ "\n",
+ " if pd.isna(sections) or len(sections) == 0 or len(sections['distance']) == 0:\n",
+ " return row\n",
+ "\n",
+ " try:\n",
+ " mode = sorted(sections['distance'].items(), key=lambda x: x[-1], reverse=True)[0][0]\n",
+ " distance = sections['distance'][mode]\n",
+ " duration = sections['duration'][mode]\n",
+ "\n",
+ " row['section_mode_argmax'] = mode\n",
+ " row['section_distance_argmax'] = distance\n",
+ " row['section_duration_argmax'] = duration\n",
+ "\n",
+ " except:\n",
+ " row['section_mode_argmax'] = np.nan\n",
+ " row['section_distance_argmax'] = np.nan\n",
+ " row['section_duration_argmax'] = np.nan\n",
+ "\n",
+ " finally:\n",
+ " return row\n",
+ " else:\n",
+ " \n",
+ " try:\n",
+ " distances = ast.literal_eval(row['section_distances'])\n",
+ " durations = ast.literal_eval(row['section_durations'])\n",
+ " modes = ast.literal_eval(row['section_modes'])\n",
+ "\n",
+ " argmax = np.argmax(distances)\n",
+ " \n",
+ " row['section_distance_argmax'] = distances[argmax]\n",
+ " row['section_duration_argmax'] = durations[argmax]\n",
+ " row['section_mode_argmax'] = modes[argmax]\n",
+ " \n",
+ " except:\n",
+ " row['section_mode_argmax'] = np.nan\n",
+ " row['section_distance_argmax'] = np.nan\n",
+ " row['section_duration_argmax'] = np.nan\n",
+ " \n",
+ " finally:\n",
+ " return row"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f0c008a3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips.reset_index(drop=True, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7e1baa06",
+ "metadata": {},
+ "source": [
+ "### Available feature generation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "de49ec4f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "available = {\n",
+ " # AllCEO\n",
+ " 'Bicycle': 'p_micro',\n",
+ " 'Do not have vehicle': 'unknown',\n",
+ " 'Do not have vehicle ': 'unknown',\n",
+ " 'Get a ride from a friend or family member': 's_car',\n",
+ " 'None': 'no_trip',\n",
+ " 'Public transportation (bus, subway, light rail, etc.)': 'transit',\n",
+ " 'Rental car (including Zipcar/ Car2Go)': 'car',\n",
+ " 'Shared bicycle or scooter': 's_micro',\n",
+ " 'Skateboard': 'p_micro',\n",
+ " 'Taxi (regular taxi, Uber, Lyft, etc)': 'ridehail',\n",
+ " 'Walk/roll': 'walk',\n",
+ " 'Prefer not to say': 'unknown',\n",
+ " # Others\n",
+ " 'public_transportation__bus__subway__ligh': 'transit',\n",
+ " 'get_a_ride_from_a_friend_or_family_membe': 's_car', \n",
+ " 'bicycle': 'p_micro', \n",
+ " 'walk': 'walk',\n",
+ " 'taxi__regular_taxi__uber__lyft__etc': 'ridehail',\n",
+ " 'rental_car__including_zipcar__car2go': 'car', \n",
+ " 'prefer_not_to_say': 'unknown'\n",
+ "}\n",
+ "\n",
+ "# We use the sensed mode to update the available modes.\n",
+ "# This is to account for any user data input errors. E.g.: user does not select car as available mode\n",
+ "# but the sensed mode is car.\n",
+ "section_mode_mapping = {\n",
+ " 'bicycling': ['p_micro', 's_micro'],\n",
+ " 'car': ['s_car', 'car', 'ridehail'],\n",
+ " 'no_sensed': ['unknown'],\n",
+ " 'walking': ['walk'],\n",
+ " 'unknown': ['unknown'],\n",
+ " 'transit': ['transit']\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "62960039",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips = filtered_trips.apply(lambda x: compute_argmax(CURRENT_DB, x), axis=1)\n",
+ "\n",
+ "# Drop all rows where argmax mode == air\n",
+ "filtered_trips.drop(\n",
+ " index=filtered_trips.loc[filtered_trips.section_mode_argmax.isin(['AIR_OR_HSR', 'air_or_hsr']),:].index, \n",
+ " inplace=True\n",
+ ")\n",
+ "\n",
+ "filtered_trips.section_mode_argmax.replace({\n",
+ " 'subway': 'transit',\n",
+ " 'no_sensed': 'unknown',\n",
+ " 'train': 'transit',\n",
+ " 'TRAM': 'transit',\n",
+ " 'LIGHT_RAIL': 'transit',\n",
+ " 'CAR': 'car',\n",
+ " 'WALKING': 'walking',\n",
+ " 'BICYCLING': 'bicycling',\n",
+ " 'UNKNOWN': 'unknown',\n",
+ " 'TRAIN': 'transit',\n",
+ " 'SUBWAY': 'transit',\n",
+ " 'BUS': 'transit',\n",
+ " 'bus': 'transit'\n",
+ "}, inplace=True)\n",
+ "\n",
+ "filtered_trips.dropna(subset='section_mode_argmax', inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8583a709",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "## Meters -> miles\n",
+ "filtered_trips['section_distance_argmax'] *= 0.000621371\n",
+ "\n",
+ "## Seconds -> minutes\n",
+ "filtered_trips['section_duration_argmax'] /= 60.\n",
+ "\n",
+ "## Total distance and duration are scaled too.\n",
+ "filtered_trips['distance'] *= 0.000621371\n",
+ "filtered_trips['duration'] /= 60."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1e4d05eb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips = filtered_trips.merge(right=filtered_survey, left_on='user_id_join', right_on='user_id_join')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "383fe251",
+ "metadata": {},
+ "source": [
+ "## Update available indicators"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ee097233",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import itertools\n",
+ "\n",
+ "new_cols = list(set(available.values()))\n",
+ "filtered_trips[new_cols] = 0\n",
+ "\n",
+ "for user_id, user_trips in filtered_trips.groupby('user_id'):\n",
+ " \n",
+ " if CURRENT_DB == \"Stage_database\":\n",
+ " \n",
+ " # Get the set of available modes (demographics.)\n",
+ " all_av_modes = user_trips['available_modes'].str.split(';').explode()\n",
+ " else:\n",
+ " # Get the set of available modes (demographics.)\n",
+ " all_av_modes = user_trips['available_modes'].str.split().explode()\n",
+ " \n",
+ " # Get all sensed modes.\n",
+ " all_sections = user_trips['section_mode_argmax'].unique()\n",
+ " \n",
+ " # Map to Common Normal Form.\n",
+ " mapped_sections = set(list(itertools.chain.from_iterable([section_mode_mapping[x] for x in all_sections])))\n",
+ " mapped_demo_av = set([available[x] for x in all_av_modes.unique()])\n",
+ " \n",
+ " # Perform a set union.\n",
+ " combined = list(mapped_sections.union(mapped_demo_av))\n",
+ " \n",
+ " # Update dummy indicators.\n",
+ " filtered_trips.loc[filtered_trips.user_id == user_id, combined] = 1\n",
+ "\n",
+ "filtered_trips.rename(columns=dict([(c, 'av_'+c) for c in new_cols]), inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "38bfcc0c",
+ "metadata": {},
+ "source": [
+ "### Cost estimation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "054a6ad1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# All values are taken from VTPI.\n",
+ "# https://www.vtpi.org/tca/tca0501.pdf\n",
+ "mode_cost_per_mile = {\n",
+ " # bicycle/skateboard\n",
+ " 'p_micro': 0.,\n",
+ " 'no_trip': 0.,\n",
+ " # Shared car is half the cost of regular car, which is $0.6/mile.\n",
+ " 's_car': 0.3,\n",
+ " # Rental car.\n",
+ " 'car': 0.6,\n",
+ " # Average of bus and train taken.\n",
+ " 'transit': 0.5,\n",
+ " # Shared bicyle or scooter - values taken from https://nacto.org/shared-micromobility-2020-2021/ and \n",
+ " # https://www.mckinsey.com/industries/automotive-and-assembly/our-insights/how-sharing-the-road-is-likely-to-transform-american-mobility\n",
+ " 's_micro': 0.3,\n",
+ " # uber/taxi/lyft\n",
+ " 'ridehail': 2.,\n",
+ " 'walk': 0.,\n",
+ " 'unknown': 0.\n",
+ "}\n",
+ "\n",
+ "# Assumptions.\n",
+ "mode_init_cost = {\n",
+ " 'p_micro': 0.,\n",
+ " 'no_trip': 0.,\n",
+ " # Shared car is half the cost of regular car, which is $0.6/mile.\n",
+ " 's_car': 0.,\n",
+ " # Rental car.\n",
+ " 'car': 0.,\n",
+ " # Average of bus and train taken.\n",
+ " 'transit': 0.,\n",
+ " # $1 unlocking cost.\n",
+ " 's_micro': 1.,\n",
+ " # uber/taxi/lyft\n",
+ " 'ridehail': 1.5,\n",
+ " 'walk': 0.,\n",
+ " 'unknown': 0.\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bccd3efb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def compute_cost_estimates(df: pd.DataFrame):\n",
+ " \n",
+ " # Create some extra colums.\n",
+ " columns = [c.replace('av_', '') for c in df.columns if 'av_' in c]\n",
+ "\n",
+ " # Initialize the columns to 0.\n",
+ " df[columns] = 0.\n",
+ "\n",
+ " rows = list()\n",
+ "\n",
+ " # Iterate over every row.\n",
+ " for _, row in df.iterrows():\n",
+ " # Check which flags are active.\n",
+ " row_dict = row.to_dict()\n",
+ "\n",
+ " # Access the section_distance_argmax attribute for the distance. Note that this is now in miles.\n",
+ " distance = row_dict['section_distance_argmax']\n",
+ " \n",
+ " # Mask using availability.\n",
+ " for lookup in columns:\n",
+ " row_dict[lookup] = row_dict['av_' + lookup] * (\n",
+ " mode_init_cost[lookup] + (mode_cost_per_mile[lookup] * distance)\n",
+ " )\n",
+ "\n",
+ " rows.append(row_dict)\n",
+ "\n",
+ " new_df = pd.DataFrame(rows)\n",
+ " new_df.rename(columns=dict([(c, 'cost_'+c) for c in columns]), inplace=True)\n",
+ "\n",
+ " return new_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c39f1901",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips = compute_cost_estimates(filtered_trips)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a6c20466",
+ "metadata": {},
+ "source": [
+ "### Outlier removal"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c05071cc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"For {CURRENT_DB=}, before outlier removal, n_rows = {filtered_trips.shape[0]}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b222715f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Drop instances where duration/distance is unusable.\n",
+ "filtered_trips.drop(\n",
+ " index=filtered_trips.loc[(filtered_trips.section_distance_argmax <= 0) | (filtered_trips.section_duration_argmax <= 0), :].index,\n",
+ " inplace=False\n",
+ ").reset_index(drop=True, inplace=True)\n",
+ "\n",
+ "\n",
+ "# bus, train, bicycling, walking, car\n",
+ "# split-apply-combine\n",
+ "def drop_outliers(df: pd.DataFrame, low=0.1, high=0.9) -> pd.DataFrame:\n",
+ " \n",
+ " def filter_by_percentiles(group):\n",
+ " distance_low = group['section_distance_argmax'].quantile(low)\n",
+ " distance_high = group['section_distance_argmax'].quantile(high)\n",
+ " duration_low = group['section_duration_argmax'].quantile(low)\n",
+ " duration_high = group['section_duration_argmax'].quantile(high)\n",
+ " \n",
+ " l1_filter = group[\n",
+ " (group['section_distance_argmax'] >= distance_low) &\n",
+ " (group['section_distance_argmax'] <= distance_high)\n",
+ " ].reset_index(drop=True)\n",
+ " \n",
+ " l2_filter = l1_filter[\n",
+ " (l1_filter['section_duration_argmax'] >= duration_low) &\n",
+ " (l1_filter['section_duration_argmax'] <= duration_high)\n",
+ " ].reset_index(drop=True)\n",
+ " \n",
+ " return l2_filter\n",
+ " \n",
+ " return df.groupby('section_mode_argmax').apply(filter_by_percentiles).reset_index(drop=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d77febb3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips = drop_outliers(filtered_trips, low=0.01, high=0.99)\n",
+ "\n",
+ "# Ideal speed. distance/time (in hours).\n",
+ "filtered_trips['mph'] = (\n",
+ " (filtered_trips['section_distance_argmax'] * 60.)/filtered_trips['section_duration_argmax']\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b52d5325",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips[['section_mode_argmax', 'section_duration_argmax', 'section_distance_argmax', 'mph']].head(10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c7ed953d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def filter_mph(df: pd.DataFrame, low=0.1, high=0.9) -> pd.DataFrame:\n",
+ " \n",
+ " MPH_THRESHOLDS = {\n",
+ " # https://www.sciencedirect.com/science/article/pii/S2210670718304682\n",
+ " 'bicycling': 15.,\n",
+ " # https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7806575/\n",
+ " 'walking': 2.93\n",
+ " }\n",
+ " \n",
+ " def custom_filter(group):\n",
+ " # Drop data specified in the dict manually.\n",
+ " if group.name in MPH_THRESHOLDS.keys():\n",
+ " f_df = group[group['mph'] <= MPH_THRESHOLDS[group.name]]\n",
+ " else:\n",
+ " mph_low = group['mph'].quantile(low)\n",
+ " mph_high = group['mph'].quantile(high)\n",
+ "\n",
+ " f_df = group[(group['mph'] >= mph_low) & (group['mph'] <= mph_high)]\n",
+ " \n",
+ " return f_df\n",
+ " \n",
+ " return df.groupby('section_mode_argmax').apply(custom_filter).reset_index(drop=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0c1904cd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips = filter_mph(filtered_trips, low=0.01, high=0.99)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3dce2b1c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips.groupby('section_mode_argmax')[['section_distance_argmax', 'section_duration_argmax']].describe()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "396f196b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips.groupby('section_mode_argmax')[['mph']].describe()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "41109148",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"For {CURRENT_DB=}, After outlier removal, n_rows = {filtered_trips.shape[0]}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1ca22a08",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "to_drop=[\n",
+ " '_id', 'additions', 'cleaned_section_summary', 'cleaned_trip', 'confidence_threshold', \n",
+ " 'end_fmt_time', 'end_loc', 'end_local_dt_day', 'raw_trip', 'purpose_confirm',\n",
+ " 'end_local_dt_minute', 'end_local_dt_month', 'end_local_dt_second', 'end_local_dt_timezone', \n",
+ " 'end_local_dt_weekday', 'end_local_dt_year', 'end_place', 'end_ts', 'expectation', 'expected_trip', \n",
+ " 'inferred_labels', 'inferred_section_summary', 'inferred_trip', 'metadata_write_ts', 'mode_confirm', \n",
+ " 'section_durations', 'section_modes', 'source', 'start_fmt_time', 'start_loc', 'start_local_dt_day', \n",
+ " 'start_local_dt_minute', 'start_local_dt_month', 'start_local_dt_second', \n",
+ " 'start_local_dt_timezone', 'start_local_dt_weekday', 'start_local_dt_year', 'start_place', \n",
+ " 'start_ts', 'user_id_join', 'user_input', 'survey_user_id', 'section_distances',\n",
+ " 'data.local_dt.year', 'data.local_dt.month', 'data.local_dt.day', 'data.local_dt.hour', \n",
+ " 'data.local_dt.minute', 'data.local_dt.second', 'data.local_dt.weekday', 'data.local_dt.timezone',\n",
+ " 'data.fmt_time'\n",
+ "]\n",
+ "\n",
+ "for col in to_drop:\n",
+ " if col in filtered_trips.columns:\n",
+ " filtered_trips.drop(columns=[col], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2937d4ef",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filtered_trips.rename({'start_local_dt_hour': 'start:hour', 'end_local_dt_hour': 'end:hour'}, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "87c7fc92",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(filtered_trips.columns.tolist())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9ea36cad",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "display(filtered_trips.head())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a7018bf4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Done processing for {CURRENT_DB=}, Number of unique users: {len(filtered_trips.user_id.unique())}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0eacc539",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "targets = ['p_micro', 'no_trip', 's_car', 'transit', 'car', 's_micro', 'ridehail', 'walk', 'unknown']\n",
+ "\n",
+ "# Rename and map targets.\n",
+ "filtered_trips.rename(columns={'replaced_mode': 'target'}, inplace=True)\n",
+ "filtered_trips.replace({'target': {t: ix+1 for ix, t in enumerate(targets)}}, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fe5a1909",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(filtered_trips.info())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31f35a04",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "savepath = Path('./data/filtered_data')\n",
+ "\n",
+ "if not savepath.exists():\n",
+ " savepath.mkdir()\n",
+ "\n",
+ "filtered_trips.to_csv(savepath / f'preprocessed_data_{CURRENT_DB}.csv', index=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "065c1911",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "emission",
+ "language": "python",
+ "name": "emission"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.16"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/replacement_mode_modeling/02_run_trip_level_models.py b/replacement_mode_modeling/02_run_trip_level_models.py
new file mode 100644
index 00000000..95f77ab4
--- /dev/null
+++ b/replacement_mode_modeling/02_run_trip_level_models.py
@@ -0,0 +1,508 @@
+from enum import Enum
+import random
+import warnings
+import argparse
+from pathlib import Path
+from collections import Counter
+
+# Math and graphing.
+import pandas as pd
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+# sklearn imports.
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import f1_score, r2_score, ConfusionMatrixDisplay
+from scipy.special import kl_div
+from sklearn.metrics import classification_report
+from sklearn.model_selection import GridSearchCV, StratifiedGroupKFold
+from pprint import pprint
+from sklearn.inspection import permutation_importance
+from time import perf_counter
+from sklearn.ensemble import RandomForestClassifier
+
+warnings.simplefilter(action='ignore', category=Warning)
+
+# Global experiment flags and variables.
+SEED = 13210
+TARGETS = ['p_micro', 'no_trip', 's_car', 'transit', 'car', 's_micro', 'ridehail', 'walk', 'unknown']
+MAP = {ix+1:t for ix, t in enumerate(TARGETS)}
+
+CV = False
+
+# Set the Numpy seed too.
+random.seed(SEED)
+np.random.seed(SEED)
+
+class SPLIT_TYPE(Enum):
+ INTRA_USER = 0
+ INTER_USER = 1
+ TARGET = 2
+ MODE = 3
+ HIDE_USER = 4
+
+
+class SPLIT(Enum):
+ TRAIN = 0
+ TEST = 1
+
+
+def get_train_test_splits(data: pd.DataFrame, how=SPLIT_TYPE, test_ratio=0.2, shuffle=True):
+
+ if how == SPLIT_TYPE.INTER_USER:
+
+ X = data.drop(columns=['target'])
+ y = data['target'].values
+ groups = data.user_id.values
+
+ # n_splits determines split size. So n=5, is 20% for each split, which is what we want.
+ splitter = StratifiedGroupKFold(n_splits=5, shuffle=shuffle, random_state=SEED)
+ # splitter = GroupKFold(n_splits=5)
+
+ for train_index, test_index in splitter.split(X, y, groups):
+ X_tr = data.iloc[train_index, :]
+ X_te = data.iloc[test_index, :]
+
+ # Iterate only once and break.
+ break
+
+ return X_tr, X_te, None
+
+ elif how == SPLIT_TYPE.INTRA_USER:
+
+ # There are certain users with only one observation. What do we do with those?
+ # As per the mobilitynet modeling pipeline, we randomly assign them to either the
+ # training or test set.
+
+ value_counts = data.user_id.value_counts()
+ single_count_ids = value_counts[value_counts == 1].index
+
+ data_filtered = data.loc[~data.user_id.isin(single_count_ids), :].reset_index(drop=True)
+ data_single_counts = data.loc[data.user_id.isin(single_count_ids), :].reset_index(drop=True)
+
+ X_tr, X_te = train_test_split(
+ data_filtered, test_size=test_ratio, shuffle=shuffle, stratify=data_filtered.user_id,
+ random_state=SEED
+ )
+
+ data_single_counts['assigned'] = np.random.choice(['train', 'test'], len(data_single_counts))
+ X_tr_merged = pd.concat(
+ [X_tr, data_single_counts.loc[data_single_counts.assigned == 'train', :].drop(
+ columns=['assigned'], inplace=False
+ )],
+ ignore_index=True, axis=0
+ )
+
+ X_te_merged = pd.concat(
+ [X_te, data_single_counts.loc[data_single_counts.assigned == 'test', :].drop(
+ columns=['assigned'], inplace=False
+ )],
+ ignore_index=True, axis=0
+ )
+
+ return X_tr_merged, X_te_merged, None
+
+ elif how == SPLIT_TYPE.TARGET:
+
+ X_tr, X_te = train_test_split(
+ data, test_size=test_ratio, shuffle=shuffle, stratify=data.target,
+ random_state=SEED
+ )
+
+ return X_tr, X_te, None
+
+ elif how == SPLIT_TYPE.MODE:
+ X_tr, X_te = train_test_split(
+ data, test_size=test_ratio, shuffle=shuffle, stratify=data.section_mode_argmax,
+ random_state=SEED
+ )
+
+ return X_tr, X_te, None
+
+
+ elif how == SPLIT_TYPE.HIDE_USER:
+ users = data.user_id.value_counts(normalize=True)
+ percentiles = users.quantile([0.25, 0.5, 0.75])
+
+ low_trip_users = users[users <= percentiles[0.25]].index
+ mid_trip_users = users[(percentiles[0.25] <= users) & (users <= percentiles[0.5])].index
+ high_trip_users = users[(percentiles[0.5] <= users) & (users <= percentiles[0.75])].index
+
+ # select one from each randomly.
+ user1 = np.random.choice(low_trip_users)
+ user2 = np.random.choice(mid_trip_users)
+ user3 = np.random.choice(high_trip_users)
+
+ print(f"Users picked: {user1}, {user2}, {user3}")
+
+ # Remove these users from the entire dataset.
+ held_out = data.loc[data.user_id.isin([user1, user2, user3]), :].reset_index(drop=True)
+ remaining = data.loc[~data.user_id.isin([user1, user2, user3]), :].reset_index(drop=True)
+
+ # Split randomly.
+ X_tr, X_te = train_test_split(
+ remaining, test_size=test_ratio, shuffle=shuffle, random_state=SEED
+ )
+
+ return X_tr, X_te, held_out
+
+ raise NotImplementedError("Unknown split type")
+
+
+def get_duration_estimate(df: pd.DataFrame, dset: SPLIT, model_dict: dict):
+
+ X_features = ['section_distance_argmax', 'mph']
+
+ if dset == SPLIT.TRAIN and model_dict is None:
+ model_dict = dict()
+
+ if dset == SPLIT.TEST and model_dict is None:
+ raise AttributeError("Expected model dict for testing.")
+
+ if dset == SPLIT.TRAIN:
+ for section_mode in df.section_mode_argmax.unique():
+ section_data = df.loc[df.section_mode_argmax == section_mode, :]
+ if section_mode not in model_dict:
+ model_dict[section_mode] = dict()
+
+ model = LinearRegression(fit_intercept=True)
+
+ X = section_data[X_features]
+ Y = section_data[['section_duration_argmax']]
+
+ model.fit(X, Y.values.ravel())
+
+ r2 = r2_score(y_pred=model.predict(X), y_true=Y.values.ravel())
+ print(f"\t-> Train R2 for {section_mode}: {r2}")
+
+ model_dict[section_mode]['model'] = model
+
+ elif dset == SPLIT.TEST:
+ for section_mode in df.section_mode_argmax.unique():
+ section_data = df.loc[df.section_mode_argmax == section_mode, :]
+ X = section_data[X_features]
+ Y = section_data[['section_duration_argmax']]
+
+ if section_mode not in model_dict.keys():
+ print(f"Inference for section={section_mode} could not be done due to lack of samples. Skipping...")
+ continue
+
+ y_pred = model_dict[section_mode]['model'].predict(X)
+ r2 = r2_score(y_pred=y_pred, y_true=Y.values.ravel())
+ print(f"\t-> Test R2 for {section_mode}: {r2}")
+
+ # Create the new columns for the duration.
+ new_columns = ['p_micro','no_trip','s_car','transit','car','s_micro','ridehail','walk','unknown']
+ df[TARGETS] = 0
+ df['temp'] = 0
+
+ for section in df.section_mode_argmax.unique():
+
+ # Cannot predict because the mode is present in test but not in train.
+ if section not in model_dict.keys():
+ df.loc[df.section_mode_argmax == section, 'temp'] = 0.
+ continue
+
+ X_section = df.loc[df.section_mode_argmax == section, X_features]
+
+ # broadcast to all columns.
+ df.loc[df.section_mode_argmax == section, 'temp'] = model_dict[section]['model'].predict(X_section)
+
+ for c in TARGETS:
+ df[c] = df['av_' + c] * df['temp']
+
+ df.drop(columns=['temp'], inplace=True)
+
+ df.rename(columns=dict([(x, 'tt_'+x) for x in TARGETS]), inplace=True)
+
+ # return model_dict, result_df
+ return model_dict, df
+
+# Some helper functions that will help ease redundancy in the code.
+
+def drop_columns(df: pd.DataFrame):
+ to_drop = ['section_mode_argmax', 'available_modes', 'user_id']
+
+ # Drop section_mode_argmax and available_modes.
+ return df.drop(
+ columns=to_drop,
+ inplace=False
+ )
+
+
+def scale_values(df: pd.DataFrame, split: SPLIT, scalers=None):
+ # Scale costs using StandardScaler.
+ costs = df[[c for c in df.columns if 'cost_' in c]].copy()
+ times = df[[c for c in df.columns if 'tt_' in c or 'duration' in c]].copy()
+ distances = df[[c for c in df.columns if 'distance' in c or 'mph' in c]].copy()
+
+ print(
+ "Cost columns to be scaled: ", costs.columns,"\nTime columns to be scaled: ", times.columns, \
+ "\nDistance columns to be scaled: ", distances.columns
+ )
+
+ if split == SPLIT.TRAIN and scalers is None:
+ cost_scaler = StandardScaler()
+ tt_scaler = StandardScaler()
+ dist_scaler = StandardScaler()
+
+ cost_scaled = pd.DataFrame(
+ cost_scaler.fit_transform(costs),
+ columns=costs.columns,
+ index=costs.index
+ )
+
+ tt_scaled = pd.DataFrame(
+ tt_scaler.fit_transform(times),
+ columns=times.columns,
+ index=times.index
+ )
+
+ dist_scaled = pd.DataFrame(
+ dist_scaler.fit_transform(distances),
+ columns=distances.columns,
+ index=distances.index
+ )
+
+ elif split == SPLIT.TEST and scalers is not None:
+
+ cost_scaler, tt_scaler, dist_scaler = scalers
+
+ cost_scaled = pd.DataFrame(
+ cost_scaler.transform(costs),
+ columns=costs.columns,
+ index=costs.index
+ )
+
+ tt_scaled = pd.DataFrame(
+ tt_scaler.transform(times),
+ columns=times.columns,
+ index=times.index
+ )
+
+ dist_scaled = pd.DataFrame(
+ dist_scaler.transform(distances),
+ columns=distances.columns,
+ index=distances.index
+ )
+
+ else:
+ raise NotImplementedError("Unknown split")
+
+ # Drop the original columns.
+ df.drop(
+ columns=costs.columns.tolist() + times.columns.tolist() + distances.columns.tolist(),
+ inplace=True
+ )
+
+ df = df.merge(right=cost_scaled, left_index=True, right_index=True)
+ df = df.merge(right=tt_scaled, left_index=True, right_index=True)
+ df = df.merge(right=dist_scaled, left_index=True, right_index=True)
+
+ return df, (cost_scaler, tt_scaler, dist_scaler)
+
+
+def train(X_tr, Y_tr):
+ if CV:
+
+ model = RandomForestClassifier(random_state=SEED)
+
+ # We want to build bootstrapped trees that would not always use all the features.
+ param_set2 = {
+ 'n_estimators': [150, 200, 250],
+ 'min_samples_split': [2, 3, 4],
+ 'min_samples_leaf': [1, 2, 3],
+ 'class_weight': ['balanced_subsample'],
+ 'max_features': [None, 'sqrt'],
+ 'bootstrap': [True]
+ }
+
+ cv_set2 = StratifiedKFold(n_splits=3, shuffle=True, random_state=SEED)
+
+ clf_set2 = GridSearchCV(model, param_set2, cv=cv_set2, n_jobs=-1, scoring='f1_weighted', verbose=1)
+
+ start = perf_counter()
+
+ clf_set2.fit(
+ X_tr,
+ Y_tr
+ )
+
+ time_req = (perf_counter() - start)/60.
+
+ best_model = clf_set2.best_estimator_
+ else:
+ best_model = RandomForestClassifier(
+ n_estimators=150,
+ max_depth=None,
+ min_samples_leaf=2,
+ bootstrap=True,
+ class_weight='balanced_subsample',
+ random_state=SEED,
+ n_jobs=-1
+ ).fit(X_tr, Y_tr)
+
+ return best_model
+
+
+def predict(model, X_tr, Y_tr, X_te, Y_te):
+
+ y_test_pred = model.predict(X_te)
+ y_train_pred = model.predict(X_tr)
+
+ train_f1 = f1_score(
+ y_true=Y_tr,
+ y_pred=y_train_pred,
+ average='weighted',
+ zero_division=0.
+ )
+
+ test_f1 = f1_score(
+ y_true=Y_te,
+ y_pred=y_test_pred,
+ average='weighted',
+ zero_division=0.
+ )
+
+ return y_train_pred, train_f1, y_test_pred, test_f1
+
+
+def run_sampled_sweep(df: pd.DataFrame, dir_name: Path, **kwargs):
+
+ targets = TARGETS.copy()
+
+ split = kwargs.pop('split', None)
+
+ try:
+ train_data, test_data, hidden_data = get_train_test_splits(data=df, how=split, shuffle=True)
+ except Exception as e:
+ print(e)
+ return
+
+ params, train_data = get_duration_estimate(train_data, SPLIT.TRAIN, None)
+ _, test_data = get_duration_estimate(test_data, SPLIT.TEST, params)
+
+ train_data = drop_columns(train_data)
+ test_data = drop_columns(test_data)
+
+ X_tr, Y_tr = train_data.drop(columns=['target'], inplace=False), train_data.target.values.ravel()
+ X_te, Y_te = test_data.drop(columns=['target'], inplace=False), test_data.target.values.ravel()
+
+ model = train(X_tr, Y_tr)
+ tr_preds, tr_f1, te_preds, te_f1 = predict(model, X_tr, Y_tr, X_te, Y_te)
+
+ print(f"\t-> Train F1: {tr_f1}, Test F1: {te_f1}")
+
+ importance = sorted(
+ zip(
+ model.feature_names_in_,
+ model.feature_importances_
+ ),
+ key=lambda x: x[-1], reverse=True
+ )
+
+ with open(dir_name / 'f1_scores.txt', 'w') as f:
+ f.write(f"Train F1: {tr_f1}\nTest F1: {te_f1}")
+
+ importance_df = pd.DataFrame(importance, columns=['feature_name', 'importance'])
+ importance_df.to_csv(dir_name / 'feature_importance.csv', index=False)
+
+ # target_names = [MAP[x] for x in np.unique(Y_te)]
+
+ with open(dir_name / 'classification_report.txt', 'w') as f:
+ f.write(classification_report(y_true=Y_te, y_pred=te_preds))
+
+ if split == SPLIT_TYPE.HIDE_USER and hidden_data is not None:
+ _, hidden_data = get_duration_estimate(hidden_data, SPLIT.TEST, params)
+ hidden_data = drop_columns(hidden_data)
+
+ X_hid, Y_hid = hidden_data.drop(columns=['target'], inplace=False), hidden_data.target.values.ravel()
+
+ tr_preds, tr_f1, te_preds, te_f1 = predict(model, X_tr, Y_tr, X_hid, Y_hid)
+ print(f"\t\t ---> Hidden user F1: {te_f1} <---")
+
+ fig, ax = plt.subplots(figsize=(7, 7))
+ cm = ConfusionMatrixDisplay.from_estimator(
+ model,
+ X=X_te,
+ y=Y_te,
+ ax=ax
+ )
+ # ax.set_xticklabels(target_names, rotation=45)
+ # ax.set_yticklabels(target_names)
+ fig.tight_layout()
+ plt.savefig(dir_name / 'test_confusion_matrix.png')
+ plt.close('all')
+
+
+def save_metadata(dir_name: Path, **kwargs):
+ with open(dir_name / 'metadata.txt', 'w') as f:
+ for k, v in kwargs.items():
+ f.write(f"{k}: {v}\n")
+
+
+
+if __name__ == "__main__":
+
+ datasets = sorted(list(Path('../data/filtered_data').glob('preprocessed_data_*.csv')))
+
+ start = perf_counter()
+
+ for dataset in datasets:
+ name = dataset.name.replace('.csv', '')
+
+ print(f"Starting modeling for dataset = {name}")
+
+ data = pd.read_csv(dataset)
+
+ if 'deprecatedID' in data.columns:
+ data.drop(columns=['deprecatedID'], inplace=True)
+ if 'data.key' in data.columns:
+ data.drop(columns=['data.key'], inplace=True)
+
+ print(f"# Samples found: {len(data)}, # unique users: {len(data.user_id.unique())}")
+
+ print("Beginning sweeps.")
+
+ # args = parse_args()
+ sweep_number = 1
+
+ root = Path('../outputs/benchmark_results')
+ if not root.exists():
+ root.mkdir()
+
+
+ if 'section_mode_argmax' in data.columns and (data.section_mode_argmax.value_counts() < 2).any():
+ # Find which mode.
+ counts = data.section_mode_argmax.value_counts()
+ modes = counts[counts < 2].index.tolist()
+ print(f"Dropping {modes} because of sparsity (<2 samples)")
+
+ data = data.loc[~data.section_mode_argmax.isin(modes), :].reset_index(drop=True)
+
+
+ for split in [SPLIT_TYPE.INTER_USER, SPLIT_TYPE.INTRA_USER, SPLIT_TYPE.TARGET, SPLIT_TYPE.MODE, SPLIT_TYPE.HIDE_USER]:
+
+ kwargs = {
+ 'dataset': name,
+ 'split': split
+ }
+
+ dir_name = root / f'benchmark_{name}_{sweep_number}'
+
+ if not dir_name.exists():
+ dir_name.mkdir()
+
+ print(f"\t-> Running sweep #{sweep_number} with metadata={str(kwargs)}")
+ save_metadata(dir_name, **kwargs)
+ run_sampled_sweep(data.copy(), dir_name, **kwargs)
+ print(f"Completed benchmarking for {sweep_number} experiment.")
+ print(50*'-')
+ sweep_number += 1
+
+ elapsed = perf_counter() - start
+
+ print(f"Completed sweeps in {elapsed/60.} minutes")
\ No newline at end of file
diff --git a/replacement_mode_modeling/03_user_level_models.ipynb b/replacement_mode_modeling/03_user_level_models.ipynb
new file mode 100644
index 00000000..1f17a91c
--- /dev/null
+++ b/replacement_mode_modeling/03_user_level_models.ipynb
@@ -0,0 +1,1243 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "04ccf092",
+ "metadata": {},
+ "source": [
+ "## Some important points to remember:\n",
+ "\n",
+ "### We want to experiment with two types of models:\n",
+ "\n",
+ "\n",
+ "1. have one row per user, so that when predicting modes for a new user, we pick the \"similar user\" or users and determine the replaced mode\n",
+ " - In this, the traditional approach would only use demographics for the user features, we may experiment with some summaries of the trip data that will function as some level of \"fingerprint\" for the user. Ideally we would be able to show that this performs better than demographics alone\n",
+ " - Note also that the original method that you had outlined where the training set is a list of trips (O()) is a third approach which we will be comparing these two against"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c0c1ee88",
+ "metadata": {},
+ "source": [
+ "Target order:\n",
+ "\n",
+ "```\n",
+ "['p_micro', 'no_trip', 's_car', 'transit', 'car', 's_micro', 'ridehail', 'walk', 'unknown']\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "21ef0f2e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import random\n",
+ "import os\n",
+ "import pickle\n",
+ "import ast\n",
+ "import matplotlib.pyplot as plt\n",
+ "import seaborn as sns\n",
+ "\n",
+ "from sklearn.linear_model import LinearRegression\n",
+ "from sklearn.ensemble import RandomForestClassifier\n",
+ "from sklearn.metrics import r2_score, f1_score, log_loss\n",
+ "from sklearn.model_selection import train_test_split, RandomizedSearchCV, StratifiedKFold, KFold\n",
+ "from sklearn.neighbors import KNeighborsClassifier\n",
+ "from sklearn.cluster import KMeans\n",
+ "from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances\n",
+ "from enum import Enum\n",
+ "from scipy.stats import uniform\n",
+ "from typing import List, Dict, Union\n",
+ "from pandas.api.types import is_numeric_dtype\n",
+ "from sklearn.manifold import TSNE\n",
+ "from multiprocessing import cpu_count\n",
+ "\n",
+ "pd.set_option('display.max_columns', 100)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fef98692",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "SEED = 13210\n",
+ "\n",
+ "np.random.seed(SEED)\n",
+ "random.seed(SEED)\n",
+ "\n",
+ "SimilarityMetric = Enum('SimilarityMetric', ['COSINE', 'EUCLIDEAN', 'KNN', 'KMEANS'])\n",
+ "GroupType = Enum('GroupType', ['GROUPBY', 'CUT'])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "79f8c51a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "DATA_SOURCE = [\n",
+ " ('./data/filtered_data/preprocessed_data_Stage_database.csv', 'allceo'),\n",
+ " ('./data/filtered_data/preprocessed_data_openpath_prod_durham.csv', 'durham'),\n",
+ " ('./data/filtered_data/preprocessed_data_openpath_prod_mm_masscec.csv', 'masscec'),\n",
+ " ('./data/filtered_data/preprocessed_data_openpath_prod_ride2own.csv', 'ride2own'),\n",
+ " ('./data/filtered_data/preprocessed_data_openpath_prod_uprm_nicr.csv', 'nicr')\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e3d9c5bd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "## CHANGE THE DB INDEX HERE.\n",
+ "DB_NUMBER = 0\n",
+ "\n",
+ "PATH = DATA_SOURCE[DB_NUMBER][0]\n",
+ "CURRENT_DB = DATA_SOURCE[DB_NUMBER][1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e37f8922",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df = pd.read_csv(PATH)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5bfa6843",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "not_needed = ['deprecatedID', 'data.key']\n",
+ "\n",
+ "for col in not_needed:\n",
+ " if col in df.columns:\n",
+ " df.drop(columns=[col], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "72793473",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "print(df.columns.tolist())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "765f08ff",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def generate_tsne_plots(df: pd.DataFrame, **kwargs):\n",
+ " \n",
+ " df = df.copy()\n",
+ " \n",
+ " # Important - if not cast as a category, seaborn considers this as a numerical value.\n",
+ " df.target = df.target.astype('category')\n",
+ " \n",
+ " # print(\"Unique targets: \", df.target.unique())\n",
+ " \n",
+ " # According to the docs, > consider choosing a perplexity between 5 and 50.\n",
+ " tsne = TSNE(\n",
+ " n_components=2,\n",
+ " perplexity=kwargs.pop('perplexity', 5),\n",
+ " n_iter=kwargs.pop('n_iter', 2000),\n",
+ " metric=kwargs.pop('metric', 'cosine'),\n",
+ " random_state=SEED,\n",
+ " n_jobs=os.cpu_count()\n",
+ " )\n",
+ " \n",
+ " if df.index.name == 'user_id':\n",
+ " df.reset_index(drop=False, inplace=True)\n",
+ " \n",
+ " if 'user_id' in df.columns:\n",
+ " df.drop(columns=['user_id'], inplace=True)\n",
+ " \n",
+ " targets = df.target.values\n",
+ " df.drop(columns=['target'], inplace=True)\n",
+ " \n",
+ " projected = tsne.fit_transform(df)\n",
+ " \n",
+ " fig, ax = plt.subplots()\n",
+ " sns.scatterplot(x=projected[:, 0], y=projected[:, 1], hue=targets, ax=ax)\n",
+ " ax.set(xlabel='Embedding dimension 1', ylabel='Embedding dimension 2', title='t-SNE plot for data')\n",
+ " plt.show()\n",
+ " \n",
+ " return projected"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cfe76e8c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_mode_coverage(df: pd.DataFrame):\n",
+ " \n",
+ " coverage_df = df.groupby(['user_id', 'section_mode_argmax']).size().unstack(fill_value=0)\n",
+ " coverage_df.columns = ['coverage_' + str(c) for c in coverage_df.columns]\n",
+ " \n",
+ " # As a preventative measure.\n",
+ " coverage_df.fillna(0, inplace=True)\n",
+ " \n",
+ " # Normalize over rows.\n",
+ " coverage_df.iloc[:, 1:] = coverage_df.iloc[:, 1:].div(coverage_df.iloc[:, 1:].sum(axis=1), axis=0)\n",
+ " \n",
+ " return coverage_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "75313008",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_trip_summaries(df: pd.DataFrame, group_key: str, feature_list: List[str], **kwargs):\n",
+ " \n",
+ " def get_feature_summaries(trip_feature: str, is_ordinal: bool = False):\n",
+ " \n",
+ " if is_numeric_dtype(df[group_key]):\n",
+ " col_prefix = f'{trip_feature}_mean_cut'\n",
+ " if not use_qcut:\n",
+ " grouper = df.groupby(['user_id', pd.cut(df[group_key], n_cuts)])[trip_feature]\n",
+ " else:\n",
+ " grouper = df.groupby(['user_id', pd.qcut(df[group_key], n_cuts)])[trip_feature]\n",
+ " else:\n",
+ " grouper = df.groupby(['user_id', group_key])[trip_feature]\n",
+ " \n",
+ " if not is_ordinal:\n",
+ " # A mean of 0 is an actual value.\n",
+ " \n",
+ " mean = grouper.mean().unstack(level=-1, fill_value=-1.)\n",
+ " \n",
+ " mean.columns = [f'{trip_feature}_mean_' + str(c) for c in mean.columns]\n",
+ " \n",
+ " # Same with percentiles - 0 is an actual value.\n",
+ " median = grouper.median().unstack(level=-1, fill_value=-1.)\n",
+ " median.columns = [f'{trip_feature}_median_' + str(c) for c in median.columns]\n",
+ " \n",
+ " iqr_df = grouper.quantile([0.25, 0.75]).unstack(level=-1)\n",
+ " iqr = (iqr_df[0.75] - iqr_df[0.25]).unstack(level=-1)\n",
+ " iqr.fillna(-1., inplace=True)\n",
+ " iqr.columns = [f'{trip_feature}_iqr_' + str(c) for c in iqr.columns]\n",
+ "\n",
+ " # Now merge.\n",
+ " merged = mean.copy()\n",
+ " merged = merged.merge(right=median, left_index=True, right_index=True)\n",
+ " merged = merged.merge(right=iqr, left_index=True, right_index=True)\n",
+ " \n",
+ " merged.fillna(-1., inplace=True)\n",
+ "\n",
+ " return merged\n",
+ " \n",
+ " # 0 is OK to indicate NaN values.\n",
+ " f_mode = grouper.apply(\n",
+ " lambda x: x.value_counts().idxmax()\n",
+ " ).unstack(fill_value=0.)\n",
+ " \n",
+ " f_mode.columns = [f'{trip_feature}_mode_' + str(c) for c in f_mode.columns]\n",
+ " f_mode.fillna(0., inplace=True)\n",
+ " \n",
+ " return f_mode\n",
+ " \n",
+ " assert group_key not in feature_list, \"Cannot perform grouping and summarization of the same feature.\"\n",
+ " \n",
+ " # Optional kwarg for number of cuts for numeric dtype grouping.\n",
+ " # Default is 3: short, medium, long trip types:\n",
+ " # For e.g., if the group key is 'section_duration', it will be cut into three equally-sized bins,\n",
+ " # However, an alternative is also present - we could use qcut() instead, which would ensure that\n",
+ " # each bin has roughly the same number of samples.\n",
+ " n_cuts = kwargs.pop('n_cuts', 3)\n",
+ " use_qcut = kwargs.pop('use_qcut', False)\n",
+ " \n",
+ " # This will be the dataframe that all subsequent features will join to.\n",
+ " feature_df = None\n",
+ " \n",
+ " for ix, feature in enumerate(feature_list):\n",
+ " is_ordinal = feature == 'start_local_dt_hour' or feature == 'end_local_dt_hour'\n",
+ " if ix == 0:\n",
+ " feature_df = get_feature_summaries(feature, is_ordinal)\n",
+ " else:\n",
+ " next_feature_df = get_feature_summaries(feature, is_ordinal)\n",
+ " feature_df = feature_df.merge(right=next_feature_df, left_index=True, right_index=True)\n",
+ " \n",
+ " return feature_df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "63617ada",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_demographic_data(df: pd.DataFrame, **trip_kwargs):\n",
+ " \n",
+ " '''\n",
+ " A method that returns a U x (D + t) matrix, where U = number of users,\n",
+ " D = number of demographic features, t (optional) = number of trip summary features.\n",
+ " \n",
+ " When use_trip_summaries=True, the 'available_modes' column is dropped in favor of\n",
+ " the already-preprocessed av_ columns. This is because we want to incorporate trip-level\n",
+ " information into the data. When the argument is False, we want to SOLELY use demographics.\n",
+ " '''\n",
+ " \n",
+ " trip_features_to_use = trip_kwargs.pop('trip_features', None)\n",
+ " trip_group_key = trip_kwargs.pop('trip_grouping', 'section_mode_argmax')\n",
+ " \n",
+ " demographics = {\n",
+ " 'allceo': [ \n",
+ " 'has_drivers_license', 'is_student', 'is_paid', 'income_category', 'n_residence_members', \n",
+ " 'n_residents_u18', 'n_residents_with_license', 'n_motor_vehicles',\n",
+ " 'has_medical_condition', 'ft_job', 'multiple_jobs', 'n_working_residents', \n",
+ " \"highest_education_Bachelor's degree\", 'highest_education_Graduate degree or professional degree', \n",
+ " 'highest_education_High school graduate or GED', 'highest_education_Less than a high school graduate', \n",
+ " 'highest_education_Prefer not to say', 'highest_education_Some college or associates degree', \n",
+ " 'primary_job_description_Clerical or administrative support', 'primary_job_description_Custodial', \n",
+ " 'primary_job_description_Education', \n",
+ " 'primary_job_description_Manufacturing, construction, maintenance, or farming', \n",
+ " 'primary_job_description_Medical/healthcare', 'primary_job_description_Other', 'gender_Man', \n",
+ " 'gender_Man;Nonbinary/genderqueer/genderfluid', 'gender_Nonbinary/genderqueer/genderfluid', \n",
+ " 'gender_Prefer not to say', 'gender_Test', 'gender_Woman', 'gender_Woman;Nonbinary/genderqueer/genderfluid', \n",
+ " 'age_16___20_years_old', 'age_1___5_years_old', 'age_21___25_years_old', 'age_26___30_years_old', \n",
+ " 'age_31___35_years_old', 'age_36___40_years_old', 'age_41___45_years_old', 'age_46___50_years_old', \n",
+ " 'age_51___55_years_old', 'age_56___60_years_old', 'age_61___65_years_old', 'age___65_years_old', \n",
+ " 'av_s_car', 'av_walk', 'av_ridehail', 'av_s_micro', 'av_transit', 'av_no_trip', 'av_car', 'av_unknown', \n",
+ " 'av_p_micro'\n",
+ " ],\n",
+ " 'durham': [\n",
+ " 'is_student', 'is_paid', 'has_drivers_license', 'n_residents_u18', 'n_residence_members', 'income_category',\n",
+ " 'n_residents_with_license', 'n_working_residents', 'n_motor_vehicles', 'has_medical_condition', 'ft_job',\n",
+ " 'multiple_jobs', 'highest_education_bachelor_s_degree', 'highest_education_graduate_degree_or_professional_degree',\n",
+ " 'highest_education_high_school_graduate_or_ged', 'highest_education_less_than_a_high_school_graduate',\n",
+ " 'highest_education_some_college_or_associates_degree', 'primary_job_description_Clerical or administrative support',\n",
+ " 'primary_job_description_Manufacturing, construction, maintenance, or farming',\n",
+ " 'primary_job_description_Other', 'primary_job_description_Prefer not to say', 'primary_job_description_Professional, Manegerial, or Technical',\n",
+ " 'primary_job_description_Sales or service', 'gender_man', 'gender_non_binary_genderqueer_gender_non_confor',\n",
+ " 'gender_prefer_not_to_say', 'gender_woman', 'age_16___20_years_old', 'age_21___25_years_old', 'age_26___30_years_old',\n",
+ " 'age_31___35_years_old', 'age_36___40_years_old', 'age_41___45_years_old', 'age_46___50_years_old',\n",
+ " 'age_51___55_years_old', 'age_56___60_years_old', 'av_unknown', 'av_no_trip', 'av_s_micro', 'av_s_car', 'av_car',\n",
+ " 'av_p_micro', 'av_walk', 'av_transit', 'av_ridehail'\n",
+ " ],\n",
+ " 'nicr': [\n",
+ " \n",
+ " 'is_student', 'is_paid', 'has_drivers_license', 'n_residents_u18', 'n_residence_members', \n",
+ " 'income_category', 'n_residents_with_license', 'n_working_residents', 'n_motor_vehicles', \n",
+ " 'has_medical_condition', 'ft_job', 'multiple_jobs', 'highest_education_bachelor_s_degree', \n",
+ " 'highest_education_high_school_graduate_or_ged', 'highest_education_prefer_not_to_say', \n",
+ " 'highest_education_some_college_or_associates_degree', \n",
+ " 'primary_job_description_Clerical or administrative support', 'primary_job_description_Other', \n",
+ " 'gender_man', 'gender_woman', 'age_16___20_years_old', 'age_21___25_years_old', 'age_26___30_years_old', \n",
+ " 'av_s_car', 'av_no_trip', 'av_s_micro', 'av_walk', 'av_unknown', 'av_p_micro', 'av_transit', 'av_car', \n",
+ " 'av_ridehail'\n",
+ " ],\n",
+ " 'masscec': [\n",
+ " 'is_student', 'is_paid', 'has_drivers_license', 'n_residents_u18', 'n_residence_members', \n",
+ " 'income_category', 'n_residents_with_license', 'n_working_residents', \n",
+ " 'n_motor_vehicles', 'has_medical_condition', 'ft_job', 'multiple_jobs', \n",
+ " 'highest_education_bachelor_s_degree', 'highest_education_graduate_degree_or_professional_degree',\n",
+ " 'highest_education_high_school_graduate_or_ged', \n",
+ " 'highest_education_less_than_a_high_school_graduate', 'highest_education_prefer_not_to_say', \n",
+ " 'highest_education_some_college_or_associates_degree', \n",
+ " 'primary_job_description_Clerical or administrative support', \n",
+ " 'primary_job_description_Manufacturing, construction, maintenance, or farming', \n",
+ " 'primary_job_description_Other', 'primary_job_description_Prefer not to say', \n",
+ " 'primary_job_description_Professional, Manegerial, or Technical', \n",
+ " 'primary_job_description_Sales or service', 'gender_man', \n",
+ " 'gender_non_binary_genderqueer_gender_non_confor', 'gender_prefer_not_to_say', 'gender_woman', \n",
+ " 'age_16___20_years_old', 'age_21___25_years_old', 'age_26___30_years_old', \n",
+ " 'age_31___35_years_old', 'age_36___40_years_old', 'age_41___45_years_old', \n",
+ " 'age_46___50_years_old', 'age_51___55_years_old', 'age_56___60_years_old', \n",
+ " 'age_61___65_years_old', 'age___65_years_old', 'av_no_trip', 'av_transit', \n",
+ " 'av_ridehail', 'av_walk', 'av_car', 'av_p_micro', 'av_unknown', 'av_s_micro', 'av_s_car'\n",
+ " ],\n",
+ " 'ride2own': [\n",
+ " 'has_drivers_license', 'is_student', 'is_paid', 'income_category', 'n_residence_members', \n",
+ " 'n_working_residents', 'n_residents_u18', 'n_residents_with_license', 'n_motor_vehicles', \n",
+ " 'has_medical_condition', 'ft_job', 'multiple_jobs', 'highest_education_bachelor_s_degree', \n",
+ " 'highest_education_graduate_degree_or_professional_degree', \n",
+ " 'highest_education_high_school_graduate_or_ged', \n",
+ " 'highest_education_less_than_a_high_school_graduate', \n",
+ " 'highest_education_some_college_or_associates_degree', 'primary_job_description_Other', \n",
+ " 'primary_job_description_Professional, Manegerial, or Technical', \n",
+ " 'primary_job_description_Sales or service', 'gender_man', \n",
+ " 'gender_non_binary_genderqueer_gender_non_confor', 'gender_woman', 'age_16___20_years_old', \n",
+ " 'age_21___25_years_old', 'age_26___30_years_old', 'age_31___35_years_old', \n",
+ " 'age_36___40_years_old', 'age_41___45_years_old', 'age_51___55_years_old', \n",
+ " 'age_56___60_years_old', 'age___65_years_old', 'av_p_micro', 'av_s_car', 'av_car', \n",
+ " 'av_ridehail', 'av_walk', 'av_transit', 'av_no_trip', 'av_s_micro', 'av_unknown'\n",
+ " ]\n",
+ " }\n",
+ " \n",
+ " # Retain only the first instance of each user and subset the columns.\n",
+ " filtered = df.groupby('user_id').first()[demographics[CURRENT_DB]]\n",
+ " \n",
+ " # Get the targets.\n",
+ " targets = df.groupby('user_id')['target'].apply(lambda x: x.value_counts().idxmax())\n",
+ " \n",
+ " filtered = filtered.merge(right=targets, left_index=True, right_index=True)\n",
+ " \n",
+ " if trip_features_to_use is None or len(trip_features_to_use) == 0:\n",
+ "# # Use the available modes as indicators.\n",
+ "# return encode_availability(filtered)\n",
+ " return filtered\n",
+ " \n",
+ " # -----------------------------------------------------------\n",
+ " # Reaching here means that we need to include trip summaries\n",
+ " # -----------------------------------------------------------\n",
+ " \n",
+ " # For every user, generate the global trip-level summaries.\n",
+ " global_aggs = df.groupby('user_id').agg({'duration': 'mean', 'distance': 'mean'})\n",
+ " \n",
+ " # coverage.\n",
+ " coverage = get_mode_coverage(df)\n",
+ " \n",
+ " # Trip-level features.\n",
+ " trip_features = get_trip_summaries(\n",
+ " df=df, \n",
+ " group_key=trip_group_key, \n",
+ " feature_list=trip_features_to_use,\n",
+ " use_qcut=trip_kwargs.pop('use_qcut', False)\n",
+ " )\n",
+ " \n",
+ " targets = df.groupby('user_id')['target'].apply(lambda x: x.value_counts().idxmax())\n",
+ " \n",
+ " trip_features = trip_features.merge(right=coverage, left_index=True, right_index=True)\n",
+ " trip_features = trip_features.merge(right=global_aggs, left_index=True, right_index=True)\n",
+ " \n",
+ " # Finally, join with availability indicators and targets.\n",
+ " trip_features = trip_features.merge(right=targets, left_index=True, right_index=True)\n",
+ " \n",
+ " return trip_features.reset_index(drop=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fedb51e8",
+ "metadata": {},
+ "source": [
+ "## Experiment 1: Only demographics"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "66421120",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "## Educated suburban woman -> \n",
+ "# An embedding where:\n",
+ "# \"highest_education_Bachelor's degree\" == 1 or 'highest_education_Graduate degree or professional degree' == 1\n",
+ "# income_category >= 4 ( + more features that define 'suburban-ness')\n",
+ "# gender_Woman == 1\n",
+ "\n",
+ "demo_df = get_demographic_data(df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "17196eaf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(demo_df.head())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4c458c1a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tsne_kwargs = {\n",
+ " 'perplexity': min(len(demo_df)-1, 6),\n",
+ " 'n_iter': 7500,\n",
+ " 'metric': 'cosine'\n",
+ "}\n",
+ "\n",
+ "# ## PLOT BY THE WAY IN WHICH PEOPLE USE THE SAME REPLACED MODE AND CHECK THE SIMILARITY.\n",
+ "\n",
+ "projections = generate_tsne_plots(demo_df, **tsne_kwargs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c023cf66",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# No stratification, pure random.\n",
+ "demo_df.reset_index(drop=False, inplace=True)\n",
+ "train, test = train_test_split(demo_df, test_size=0.2, random_state=SEED)\n",
+ "\n",
+ "TRAIN_USERS = train.user_id.unique().tolist()\n",
+ "TEST_USERS = test.user_id.unique().tolist()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "376a4391",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(train.shape[0], test.shape[0])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "630d6c08",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Ensuring that no user information is leaked across sets.\n",
+ "assert train.shape[0] + test.shape[0] == len(df.user_id.unique())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ef77c9c8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def evaluate_using_similarity(test_df, train_df, metric=SimilarityMetric.COSINE, **metric_kwargs):\n",
+ " \n",
+ " '''\n",
+ " This method treats each user row as a 'fingerprint' (embedding vector). We assume that we\n",
+ " have no idea about the test set labels. To find which replaced mode is most likely for the test\n",
+ " users, we compute the cosine similarity of each test user against the users in the training set.\n",
+ " For the most similar user, we use their target as a proxy for the test user's replaced mode.\n",
+ " This operates on the following intuition: If User A and User B are similar, then their replaced\n",
+ " modes are also similar.\n",
+ " '''\n",
+ " \n",
+ " tr_targets = train_df.target.values\n",
+ " tr = train_df.drop(columns=['target', 'user_id'], inplace=False).reset_index(drop=True, inplace=False)\n",
+ " \n",
+ " te_targets = test_df.target.values\n",
+ " te = test_df.drop(columns=['target', 'user_id'], inplace=False).reset_index(drop=True, inplace=False)\n",
+ " \n",
+ " if metric == SimilarityMetric.COSINE:\n",
+ " # Use cosine similarity to determine which element in the train set this user is closest to.\n",
+ " # Offset the columns from the second entry to exclude the user_id column.\n",
+ " # Returns a (n_te, n_tr) matrix.\n",
+ " sim = cosine_similarity(te.values, tr.values)\n",
+ " \n",
+ " # Compute the argmax across the train set.\n",
+ " argmax = np.argmax(sim, axis=1)\n",
+ "\n",
+ " # Index into the training targets to retrieve predicted label.\n",
+ " y_test_pred = tr_targets[argmax]\n",
+ " \n",
+ " elif metric == SimilarityMetric.EUCLIDEAN:\n",
+ " \n",
+ " # Here, we choose the embedding with the smallest L2 distance.\n",
+ " distances = euclidean_distances(te.values, tr.values)\n",
+ " \n",
+ " # We choose argmin\n",
+ " argmin = np.argmin(distances, axis=1)\n",
+ " \n",
+ " # Index into the targets.\n",
+ " y_test_pred = tr_targets[argmin]\n",
+ " \n",
+ " elif metric == SimilarityMetric.KNN:\n",
+ " \n",
+ " n_neighbors = metric_kwargs.pop('n_neighbors', 3)\n",
+ " \n",
+ " if n_neighbors >= len(tr):\n",
+ " return -1.\n",
+ " \n",
+ " # Build the KNN classifier. By default, let it be 3.\n",
+ " knn = KNeighborsClassifier(\n",
+ " n_neighbors=n_neighbors,\n",
+ " weights='distance',\n",
+ " metric=metric_kwargs.pop('knn_metric', 'cosine'),\n",
+ " n_jobs=os.cpu_count()\n",
+ " )\n",
+ " \n",
+ " # Fit the data to the KNN model\n",
+ " knn.fit(tr, tr_targets)\n",
+ " \n",
+ " y_test_pred = knn.predict(te)\n",
+ " \n",
+ " elif metric == SimilarityMetric.KMEANS:\n",
+ " \n",
+ " n_clusters = metric_kwargs.pop('n_clusters', 8)\n",
+ " \n",
+ " if n_clusters >= len(tr):\n",
+ " return -1\n",
+ " \n",
+ " # Build the model.\n",
+ " kmeans = KMeans(\n",
+ " n_clusters=n_clusters,\n",
+ " max_iter=metric_kwargs.pop('max_iter', 300),\n",
+ " n_init='auto',\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " # Fit the clustering model\n",
+ " kmeans.fit(tr)\n",
+ " \n",
+ " # Construct the auxiliary df and merge with the training set.\n",
+ " label_df = pd.DataFrame({'label': kmeans.labels_, 'target': tr_targets}, index=tr.index)\n",
+ " \n",
+ " # Now, perform an inference on the test set.\n",
+ " predicted_labels = kmeans.predict(te)\n",
+ " \n",
+ " y_test_pred = []\n",
+ " for prediction in predicted_labels:\n",
+ " most_likely = label_df.loc[label_df.label == prediction, 'target'].value_counts().idxmax()\n",
+ " y_test_pred.append(most_likely)\n",
+ " \n",
+ " else:\n",
+ " raise NotImplementedError(\"Unknown similarity metric\")\n",
+ " \n",
+ " \n",
+ " f1 = f1_score(y_true=te_targets, y_pred=y_test_pred, average='weighted')\n",
+ " print(f\"Test F1 score using {metric.name} = {f1}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1a95ad5e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for metric in [\n",
+ " SimilarityMetric.COSINE, SimilarityMetric.EUCLIDEAN, SimilarityMetric.KNN, SimilarityMetric.KMEANS\n",
+ "]:\n",
+ " evaluate_using_similarity(test, train, metric, n_clusters=3)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "16e435a6",
+ "metadata": {},
+ "source": [
+ "Not bad - using just a simple random split gives us the following results:\n",
+ "\n",
+ "$allCEO$:\n",
+ "\n",
+ "```\n",
+ "Test F1 score using COSINE = 0.42692939244663386\n",
+ "Test F1 score using EUCLIDEAN = 0.4126984126984127\n",
+ "Test F1 score using KNN = 0.4393241167434716\n",
+ "Test F1 score using KMEANS = 0.4733893557422969\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "81f0e842",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def custom_nll_scorer(clf, X, y):\n",
+ " \n",
+ " # [[yp1, yp2, yp3, ...], [yp1, yp3, ...]]\n",
+ " y_pred = clf.predict_proba(X)\n",
+ " \n",
+ " return -log_loss(y_true=y, y_pred=y_pred, labels=sorted(np.unique(y)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a3a6af8f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def estimate_using_model(train, test, **model_kwargs):\n",
+ " \n",
+ " cv = model_kwargs.pop('cv', None)\n",
+ " n_splits = model_kwargs.pop('n_splits', 5)\n",
+ " n_iter = model_kwargs.pop('n_iter', 500)\n",
+ " \n",
+ " if cv is None:\n",
+ " # Define the train-val splitter.\n",
+ " cv = KFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n",
+ " \n",
+ " params = {\n",
+ " 'n_estimators': np.arange(100, 1001, 50),\n",
+ " 'max_depth': [i for i in range(5, 101, 5)],\n",
+ " 'ccp_alpha': np.linspace(0, 1, 10),\n",
+ " 'class_weight': ['balanced', 'balanced_subsample', None],\n",
+ " 'min_samples_split': np.arange(2, 25, 2),\n",
+ " 'min_samples_leaf': np.arange(1, 25)\n",
+ " }\n",
+ " \n",
+ " rf = RandomForestClassifier(random_state=SEED)\n",
+ " \n",
+ " # Search over hparams to minimize negative log likelihood. \n",
+ "# clf = RandomizedSearchCV(\n",
+ "# rf, params, n_iter=n_iter, scoring=custom_nll_scorer, \n",
+ "# n_jobs=os.cpu_count(), cv=cv, random_state=SEED,\n",
+ "# verbose=0\n",
+ "# )\n",
+ " \n",
+ " clf = RandomizedSearchCV(\n",
+ " rf, params, n_iter=n_iter, scoring='f1_weighted', \n",
+ " n_jobs=cpu_count(), cv=cv, random_state=SEED,\n",
+ " verbose=0\n",
+ " )\n",
+ " \n",
+ " X_tr = train.drop(columns=['user_id', 'target'])\n",
+ " y_tr = train.target.values.ravel()\n",
+ " \n",
+ " scorer = clf.fit(X_tr, y_tr)\n",
+ " \n",
+ " best_model = scorer.best_estimator_\n",
+ " \n",
+ " print(f\"Best val score = {scorer.best_score_}\")\n",
+ " \n",
+ " X_te = test.drop(columns=['user_id', 'target'])\n",
+ " \n",
+ " # Use the best model to compute F1 on the test set.\n",
+ " test_f1 = f1_score(y_true=test.target.values, y_pred=best_model.predict(X_te), average='weighted')\n",
+ " \n",
+ " print(f\"Test F1 = {test_f1}\")\n",
+ " \n",
+ " return best_model"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "45fef6d1",
+ "metadata": {},
+ "source": [
+ "### Uncomment to run the model "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2fab93ed",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# model = estimate_using_model(train, test)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2988c1b2",
+ "metadata": {},
+ "source": [
+ "Interesting! The model is slightly on par with K-Means!"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c6b77353",
+ "metadata": {},
+ "source": [
+ "## Experiment 2: Demographics with trip summaries"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bf7753d4",
+ "metadata": {},
+ "source": [
+ "Now that we've performed experiments with solely demographic data, let's expand the feature set by including \n",
+ "trip summary statistics. We would like this approach to do better than the aforementioned baselines."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1d46ab0f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "demo_plus_trips = get_demographic_data(\n",
+ " df, \n",
+ " trip_features=['mph', 'section_duration_argmax', 'section_distance_argmax', 'start_local_dt_hour', 'end_local_dt_hour']\n",
+ ")\n",
+ "\n",
+ "demo_plus_trips.fillna(0., inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "11c1ea2c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "demo_plus_trips.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6159c90a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train = demo_plus_trips.loc[demo_plus_trips.user_id.isin(TRAIN_USERS), :]\n",
+ "test = demo_plus_trips.loc[demo_plus_trips.user_id.isin(TEST_USERS), :]\n",
+ "\n",
+ "print(train.shape[0], test.shape[0])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "06e85bdd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for metric in [\n",
+ " SimilarityMetric.COSINE, SimilarityMetric.EUCLIDEAN, SimilarityMetric.KNN, SimilarityMetric.KMEANS\n",
+ "]:\n",
+ " evaluate_using_similarity(test, train, metric, n_clusters=4)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ba795489",
+ "metadata": {},
+ "source": [
+ "Great! Some improvement here and there.\n",
+ "\n",
+ "$allCEO$\n",
+ "```\n",
+ "Test F1 score using COSINE = 0.32098765432098775\n",
+ "Test F1 score using EUCLIDEAN = 0.36684303350970027\n",
+ "Test F1 score using KNN = 0.41269841269841273\n",
+ "Test F1 score using KMEANS = 0.4877344877344878\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "85483fc4",
+ "metadata": {},
+ "source": [
+ "### Uncomment this to run the model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9acd4b0b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Now, we try with the model\n",
+ "# estimate_using_model(train, test)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cd94c548",
+ "metadata": {},
+ "source": [
+ "Great! Compared to the previous model, we see definite improvements! I'm sure we can squeeze some more juice out of the models using fancy optimization, but as a baseline, these are good enough.\n",
+ "\n",
+ "\n",
+ "So, to recap:\n",
+ "$F1_{cosine} = 0.37$, $F1_{euclidean} = 0.33$, $F1_{knn} = 0.3$, $F1_{kmeans} = 0.36$, $F1_{RF} = 0.4215$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8a8f6491",
+ "metadata": {},
+ "source": [
+ "### Different groupings."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9ce90367",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# trip_features = ['mph', 'section_duration_argmax', 'section_distance_argmax', 'start:hour', 'end:hour']\n",
+ "\n",
+ "# for group_mode in ['section_mode_argmax', 'section_distance_argmax', 'section_duration_argmax', 'duration', 'distance']:\n",
+ " \n",
+ "# if group_mode in trip_features:\n",
+ "# _ = trip_features.pop(trip_features.index(group_mode))\n",
+ " \n",
+ "# exp_df = get_demographic_data(\n",
+ "# df, \n",
+ "# trip_grouping=group_mode,\n",
+ "# trip_features=trip_features,\n",
+ "# use_qcut=True\n",
+ "# )\n",
+ " \n",
+ "# train, test = train_test_split(exp_df, test_size=0.2, random_state=SEED)\n",
+ " \n",
+ "# for sim in [\n",
+ "# SimilarityMetric.COSINE, SimilarityMetric.EUCLIDEAN, SimilarityMetric.KNN, SimilarityMetric.KMEANS\n",
+ "# ]:\n",
+ "# evaluate_using_similarity(test, train, sim, n_clusters=3)\n",
+ " \n",
+ "# # estimate_using_model(train, test, n_iter=200)\n",
+ " \n",
+ "# print(50*'=')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6d53f945",
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "_ = generate_tsne_plots(\n",
+ " demo_plus_trips, \n",
+ " perplexity=min(len(demo_plus_trips)-1, 6), \n",
+ " n_iter=7500\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c339fcc6",
+ "metadata": {},
+ "source": [
+ "# (Experimental) Multi-level modeling\n",
+ "\n",
+ "## The code below onwards is not tested."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "213676ec",
+ "metadata": {},
+ "source": [
+ "In this approach, we want to piece together the similarity search and modeling processes. Here's a rough sketch of how it should be implemented:\n",
+ "\n",
+ "1. For every user in the training set, build a model using their entire trip history.\n",
+ "2. Consolidate these user-level models in data structure, preferably a dictionary.\n",
+ "3. Now, when we want to perform inference on a new user with no prior trips, we use the similarity search to get the user ID in the training set who is the most similar to the user in question.\n",
+ "4. We retrieve the model for this corresponding user and perform an inference. The hypothesis is that since the two users are similar, their trip substitution patterns are also similar."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c48ee430",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# def drop_columns(df: pd.DataFrame):\n",
+ "# to_drop = [\n",
+ "# 'source', 'end_ts', 'end_fmt_time', 'end_loc', 'raw_trip', 'start_ts', \n",
+ "# 'start_fmt_time', 'start_loc', 'duration', 'distance', 'start_place', \n",
+ "# 'end_place', 'cleaned_trip', 'inferred_labels', 'inferred_trip', 'expectation',\n",
+ "# 'confidence_threshold', 'expected_trip', 'user_input', 'start:year', 'start:month', \n",
+ "# 'start:day', 'start_local_dt_minute', 'start_local_dt_second', \n",
+ "# 'start_local_dt_weekday', 'start_local_dt_timezone', 'end:year', 'end:month', 'end:day', \n",
+ "# 'end_local_dt_minute', 'end_local_dt_second', 'end_local_dt_weekday', \n",
+ "# 'end_local_dt_timezone', '_id', 'metadata_write_ts', 'additions', \n",
+ "# 'mode_confirm', 'purpose_confirm', 'Mode_confirm', 'Trip_purpose', \n",
+ "# 'original_user_id', 'program', 'opcode', 'Timestamp', 'birth_year', \n",
+ "# 'available_modes', 'section_coordinates_argmax', 'section_mode_argmax'\n",
+ "# ]\n",
+ " \n",
+ "# # Drop section_mode_argmax and available_modes.\n",
+ "# return df.drop(\n",
+ "# columns=to_drop, \n",
+ "# inplace=False\n",
+ "# )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ca9e6e6a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# def construct_model_dictionary(train: pd.DataFrame):\n",
+ " \n",
+ "# def train_on_user(user_id: str):\n",
+ "# '''\n",
+ "# Given the training set and the user ID to query, filter the dataset and\n",
+ "# retain only the relevant trips. Then, create folds and optimize a model for this user.\n",
+ "# Return the trained model instance.\n",
+ "# '''\n",
+ " \n",
+ "# user_data = train.loc[train.user_id == user_id, :].reset_index(drop=True)\n",
+ " \n",
+ "# # Split user trips into train-test folds.\n",
+ "# u_train, u_test = train_test_split(user_data, test_size=0.2, shuffle=True, random_state=SEED)\n",
+ " \n",
+ "# user_model = estimate_using_model(\n",
+ "# u_train, u_test, \n",
+ "# n_iter=100\n",
+ "# )\n",
+ " \n",
+ "# return user_model\n",
+ " \n",
+ "# for user in train.user_id.unique():\n",
+ "# MODEL_DICT[user]['warm_start'] = train_on_user(user)\n",
+ "# print(50*'=')\n",
+ " \n",
+ "# print(\"\\nDone!\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2a035c16",
+ "metadata": {},
+ "source": [
+ "## Warm start:\n",
+ "\n",
+ "If the queried user has prior trips, we know that we we can harness the additional information. So if we encounter such a user, we will first find the most similar user (using only demographics). Once the most similar user is found, we query the trip model for the user and run inference through it.\n",
+ "\n",
+ "## Cold start:\n",
+ "\n",
+ "If the queried user has no prior trips, we will use the demo-only model. We first perform a similarity search and then run user inference through the demo-only model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "082c4e39",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# class MultiLevelModel:\n",
+ "# def __init__(self, model_dict: Dict, train: pd.DataFrame, test: pd.DataFrame, **model_kwargs):\n",
+ " \n",
+ "# self._demographics = [\n",
+ "# 'primary_job_commute_time', 'income_category', 'n_residence_members', 'n_residents_u18', \n",
+ "# 'n_residents_with_license', 'n_motor_vehicles', 'available_modes', 'age', 'gender_Man', \n",
+ "# 'gender_Man;Nonbinary/genderqueer/genderfluid', 'gender_Nonbinary/genderqueer/genderfluid', \n",
+ "# 'gender_Prefer not to say', 'gender_Woman', 'gender_Woman;Nonbinary/genderqueer/genderfluid', \n",
+ "# 'has_drivers_license_No', 'has_drivers_license_Prefer not to say', 'has_drivers_license_Yes', \n",
+ "# 'has_multiple_jobs_No', 'has_multiple_jobs_Prefer not to say', 'has_multiple_jobs_Yes', \n",
+ "# \"highest_education_Bachelor's degree\", 'highest_education_Graduate degree or professional degree', \n",
+ "# 'highest_education_High school graduate or GED', 'highest_education_Less than a high school graduate', \n",
+ "# 'highest_education_Prefer not to say', 'highest_education_Some college or associates degree', \n",
+ "# 'primary_job_type_Full-time', 'primary_job_type_Part-time', 'primary_job_type_Prefer not to say', \n",
+ "# 'primary_job_description_Clerical or administrative support', 'primary_job_description_Custodial', \n",
+ "# 'primary_job_description_Education', 'primary_job_description_Food service', \n",
+ "# 'primary_job_description_Manufacturing, construction, maintenance, or farming', \n",
+ "# 'primary_job_description_Medical/healthcare', 'primary_job_description_Other', \n",
+ "# 'primary_job_description_Professional, managerial, or technical', \n",
+ "# 'primary_job_description_Sales or service', 'primary_job_commute_mode_Active transport', \n",
+ "# 'primary_job_commute_mode_Car transport', 'primary_job_commute_mode_Hybrid', \n",
+ "# 'primary_job_commute_mode_Public transport', 'primary_job_commute_mode_Unknown', \n",
+ "# 'primary_job_commute_mode_WFH', 'is_overnight_trip', 'n_working_residents'\n",
+ "# ]\n",
+ " \n",
+ "# assert all([c in test.columns for c in self._demographics]), \"[test] Demographic features are missing!\"\n",
+ "# assert all([c in train.columns for c in self._demographics]), \"[train] Demographic features are missing!\"\n",
+ " \n",
+ "# self._mdict = model_dict\n",
+ "# self._train = train\n",
+ "# self._test = test\n",
+ "# self.metric = model_kwargs.pop('metric', SimilarityMetric.COSINE)\n",
+ " \n",
+ " \n",
+ "# def _phase1(self):\n",
+ " \n",
+ "# tr = self._train.copy()\n",
+ "# te = self._test.copy()\n",
+ " \n",
+ "# if tr.columns.isin(['user_id', 'target']).sum() == 2:\n",
+ "# tr = tr.drop(columns=['user_id', 'target']).reset_index(drop=True)\n",
+ " \n",
+ "# if te.columns.isin(['user_id', 'target']).sum() == 2:\n",
+ "# te = te.drop(columns=['user_id', 'target']).reset_index(drop=True)\n",
+ "\n",
+ "# te_users = self._test.user_id.tolist()\n",
+ "\n",
+ "# if self.metric == SimilarityMetric.COSINE:\n",
+ "\n",
+ "# sim = cosine_similarity(te.values, tr.values)\n",
+ "\n",
+ "# # Compute the argmax across the train set.\n",
+ "# argmax = np.argmax(sim, axis=1)\n",
+ "\n",
+ "# # Retrieve the user_id at these indices.\n",
+ "# train_users = self._train.loc[argmax, 'user_id']\n",
+ "\n",
+ "# elif self.metric == SimilarityMetric.EUCLIDEAN:\n",
+ "\n",
+ "# sim = euclidean_distances(te.values, tr.values)\n",
+ "\n",
+ "# # Compute the argmin here!\n",
+ "# argmin = np.argmin(sim, axis=1)\n",
+ "\n",
+ "# # Retrieve the train user_ids.\n",
+ "# train_users = self._train.loc[argmin, 'user_id']\n",
+ "\n",
+ "# return pd.DataFrame({'test_user_id': te_users, 'train_user_id': train_users})\n",
+ " \n",
+ " \n",
+ "# def _phase2(self, sim_df: pd.DataFrame, cold_start: bool):\n",
+ " \n",
+ "# prediction_df = list()\n",
+ " \n",
+ "# # Now, we use the sim_df to run inference based on whether \n",
+ "# for ix, row in sim_df.iterrows():\n",
+ "# train_user = row['train_user_id']\n",
+ " \n",
+ "# # Retrieve the appropriate model.\n",
+ "# user_models = self._mdict.get(train_user, None)\n",
+ " \n",
+ "# start_type = 'cold_start' if cold_start else 'warm_start'\n",
+ " \n",
+ "# # which specific model?\n",
+ "# sp_model = user_models.get(start_type, None)\n",
+ " \n",
+ "# # Now get the test user data.\n",
+ "# test_user = row['test_user_id']\n",
+ " \n",
+ "# if cold_start:\n",
+ "# test_data = self._test.loc[self._test.user_id == test_user, self._demographics]\n",
+ "# test_data = test_data.iloc[0, :]\n",
+ "# else:\n",
+ "# test_data = self._test.loc[self._test.user_id == test_user, :]\n",
+ " \n",
+ "# predictions = sp_model.predict(test_data)\n",
+ " \n",
+ "# print(f\"test: [{test_user}], predictions: {predictions}\")\n",
+ " \n",
+ " \n",
+ "# def execute_pipeline(self, cold_start: bool = False):\n",
+ "# # For each test user, get the most similar train user.\n",
+ "# sim_df = self._phase1()\n",
+ " \n",
+ "# predictions = self._phase2(sim_df, cold_start)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eb63632d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # FULL DATA.\n",
+ "# train = df.loc[df.user_id.isin(TRAIN_USERS), :]\n",
+ "# test = df.loc[df.user_id.isin(TEST_USERS), :]\n",
+ "\n",
+ "# train_counts = train.user_id.value_counts()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f2528eaa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ## We only want to train on users who have a good number of trips.\n",
+ "# good_users = train_counts[train_counts >= 100].index\n",
+ "\n",
+ "# bad_users = train_counts[train_counts < 100].index\n",
+ "\n",
+ "# print(f\"Number of users filtered out of training: {len(bad_users)}\")\n",
+ "\n",
+ "# filtered_train = train.loc[train.user_id.isin(good_users), :]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bae55b21",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Full data.\n",
+ "\n",
+ "# train_df = drop_columns(filtered_train)\n",
+ "# test_df = drop_columns(test)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "88d0e2d2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# print(train_df.shape, test_df.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "37febd6d",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# model_dict = construct_model_dictionary(train_df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b1249925",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "emission",
+ "language": "python",
+ "name": "emission"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.16"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/replacement_mode_modeling/04_FeatureClustering.ipynb b/replacement_mode_modeling/04_FeatureClustering.ipynb
new file mode 100644
index 00000000..0c222fcf
--- /dev/null
+++ b/replacement_mode_modeling/04_FeatureClustering.ipynb
@@ -0,0 +1,1166 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "789df947",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import random\n",
+ "import os\n",
+ "import itertools\n",
+ "import pickle\n",
+ "import ast\n",
+ "import matplotlib.pyplot as plt\n",
+ "import matplotlib.colors as mcolors\n",
+ "import seaborn as sns\n",
+ "\n",
+ "from pathlib import Path\n",
+ "from sklearn.linear_model import LinearRegression\n",
+ "from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances\n",
+ "from sklearn.metrics import davies_bouldin_score, calinski_harabasz_score, silhouette_score\n",
+ "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
+ "from typing import List, Dict, Union\n",
+ "from pandas.api.types import is_numeric_dtype\n",
+ "from sklearn.cluster import DBSCAN, KMeans\n",
+ "from collections import Counter\n",
+ "\n",
+ "pd.set_option('display.max_columns', None)\n",
+ "\n",
+ "%matplotlib inline\n",
+ "\n",
+ "SEED = 13210\n",
+ "\n",
+ "np.random.seed(SEED)\n",
+ "random.seed(SEED)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "aea4dda7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "DATA_SOURCE = [\n",
+ " ('./data/filtered_data/preprocessed_data_Stage_database.csv', 'allceo'),\n",
+ " ('./data/filtered_data/preprocessed_data_openpath_prod_durham.csv', 'durham'),\n",
+ " ('./data/filtered_data/preprocessed_data_openpath_prod_mm_masscec.csv', 'masscec'),\n",
+ " ('./data/filtered_data/preprocessed_data_openpath_prod_ride2own.csv', 'ride2own'),\n",
+ " ('./data/filtered_data/preprocessed_data_openpath_prod_uprm_nicr.csv', 'nicr')\n",
+ "]\n",
+ "\n",
+ "# Switch between 0-4\n",
+ "DB_NUMBER = 0"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "33ef3275",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Change this name to something unique\n",
+ "PATH = DATA_SOURCE[DB_NUMBER][0]\n",
+ "CURRENT_DB = DATA_SOURCE[DB_NUMBER][1]\n",
+ "\n",
+ "OUTPUT_DIR = Path('./outputs')\n",
+ "\n",
+ "if not OUTPUT_DIR.exists():\n",
+ " OUTPUT_DIR.mkdir()\n",
+ "\n",
+ "df = pd.read_csv(PATH)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d6f69976",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "not_needed = ['deprecatedID', 'data.key']\n",
+ "\n",
+ "for col in not_needed:\n",
+ " if col in df.columns:\n",
+ " df.drop(columns=[col], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b2281bdc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df.rename(\n",
+ " columns={'end_local_dt_hour': 'end:hour', 'start_local_dt_hour': 'start:hour', 'replaced_mode': 'target'}, \n",
+ " inplace=True\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9c22d6ac",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "TARGETS = ['p_micro', 'no_trip', 's_car', 'transit', 'car', 's_micro', 'ridehail', 'walk', 'unknown']\n",
+ "MAP = {ix+1: t for (ix, t) in enumerate(TARGETS)}\n",
+ "TARGET_MAP = {v:k for k, v in MAP.items()}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "063f6124",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df.replace({'target': TARGET_MAP}, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cef8d45b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# % of trips per mode.\n",
+ "trip_percents = df.groupby(['user_id'])['section_mode_argmax'].apply(lambda x: x.value_counts(normalize=True)).unstack(level=-1)\n",
+ "trip_percents.fillna(0., inplace=True)\n",
+ "\n",
+ "trip_percents.columns = ['coverage_'+x for x in trip_percents.columns]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "68c6af2d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "n_trips = pd.DataFrame(df.groupby('user_id').size(), columns=['n_trips'])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eff378a7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "most_common_start = df.groupby('user_id')['start:hour'].apply(lambda x: x.value_counts().idxmax())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cffbd401",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "most_common_end = df.groupby('user_id')['end:hour'].apply(lambda x: x.value_counts().idxmax())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f1eb1633",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# % of distance in each primary sensed mode.\n",
+ "total_distance = df.groupby(['user_id', 'section_mode_argmax'])['section_distance_argmax'].sum().unstack(level=-1)\n",
+ "total_distance = total_distance.div(total_distance.sum(axis=1), axis=0)\n",
+ "total_distance.fillna(0., inplace=True)\n",
+ "total_distance.columns = ['pct_distance_' + x for x in total_distance.columns]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d9cc0a0f",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "figure1_df = trip_percents.merge(right=total_distance, left_index=True, right_index=True).merge(\n",
+ " right=n_trips, left_index=True, right_index=True\n",
+ ").merge(\n",
+ " right=most_common_start, left_index=True, right_index=True\n",
+ ").merge(right=most_common_end, left_index=True, right_index=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "750fbd0c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Normalize the last three columns.\n",
+ "\n",
+ "def min_max_normalize(col: pd.Series):\n",
+ " _max, _min = col.max(), col.min()\n",
+ " return pd.Series((col - _min)/(_max - _min))\n",
+ "\n",
+ "figure1_df['n_trips'] = min_max_normalize(figure1_df['n_trips'])\n",
+ "figure1_df['start:hour'] = np.sin(figure1_df['start:hour'].values)\n",
+ "figure1_df['end:hour'] = np.sin(figure1_df['end:hour'].values)\n",
+ "\n",
+ "figure1_df.fillna(0., inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1c3d1849",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "figure1_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "aa9f5a04",
+ "metadata": {},
+ "source": [
+ "### Uncomment the following if you want to find the best eps."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "598d82bc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# epsilons = np.linspace(1e-3, 1., 1000)\n",
+ "\n",
+ "# best_eps = -np.inf\n",
+ "# best_score = -np.inf\n",
+ "\n",
+ "# for eps in epsilons:\n",
+ "# model = DBSCAN(eps=eps).fit(figure1_df)\n",
+ " \n",
+ "# if len(np.unique(model.labels_)) < 2:\n",
+ "# continue\n",
+ " \n",
+ "# score = silhouette_score(figure1_df, model.labels_)\n",
+ "# if score > best_score:\n",
+ "# best_eps = eps\n",
+ "# best_score = score\n",
+ "\n",
+ "# print(best_eps)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bc89a42d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "'''\n",
+ "AlLCEO: eps=0.542\n",
+ "durham: eps=0.661\n",
+ "masscec: eps=0.64\n",
+ "'''\n",
+ "\n",
+ "clustering = DBSCAN(eps=0.8).fit(figure1_df)\n",
+ "\n",
+ "print(Counter(clustering.labels_))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "05c9a7c4",
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "# After clustering, we would like to see what the replaced mode argmax distribution in each cluster is.\n",
+ "\n",
+ "labels = clustering.labels_\n",
+ "\n",
+ "for cix in np.unique(labels):\n",
+ " cluster_users = figure1_df.iloc[labels == cix,:].index\n",
+ " \n",
+ " print(f\"{len(cluster_users)} users in cluster {cix}\")\n",
+ " \n",
+ " # Now, for each user, look at the actual data and determine the replaced mode argmax distribution.\n",
+ " sub_df = df.loc[df.user_id.isin(cluster_users), :].reset_index(drop=True)\n",
+ " \n",
+ " sub_df['target'] = sub_df['target'].apply(lambda x: MAP[x])\n",
+ " \n",
+ " rm_argmax = sub_df.groupby('user_id')['target'].apply(lambda x: x.value_counts().idxmax())\n",
+ " fig, ax = plt.subplots()\n",
+ " rm_argmax.hist(ax=ax)\n",
+ " ax.set_title(f\"Replaced mode argmax distribution for users in cluster {cix}\")\n",
+ " ax.set_xlabel(\"Target\")\n",
+ " \n",
+ " plt.savefig(OUTPUT_DIR / f'{CURRENT_DB}__FIG1_cluster_{cix}_target_dist.png', dpi=300)\n",
+ " \n",
+ " plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f2e8e117",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "user_target_pct = pd.DataFrame()\n",
+ "\n",
+ "# For every user, compute the replaced mode distribution.\n",
+ "for user_id, user_data in df.groupby('user_id'):\n",
+ " \n",
+ " target_distribution = user_data['target'].value_counts(normalize=True)\n",
+ " target_distribution.rename(index=MAP, inplace=True)\n",
+ " user_target_pct = pd.concat([user_target_pct, target_distribution.to_frame(user_id).T])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "99369dba",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "user_target_pct.columns = ['pct_trips_' + str(x) for x in user_target_pct.columns]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6cca3671",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "target_distance = pd.DataFrame()\n",
+ "\n",
+ "# For every user, compute the replaced mode distribution.\n",
+ "for user_id, user_data in df.groupby('user_id'):\n",
+ " \n",
+ " # total_distance = user_data['distance'].sum()\n",
+ " distance_per_target = user_data.groupby('target')['section_distance_argmax'].sum()\n",
+ " distance_per_target.rename(index=MAP, inplace=True)\n",
+ " row = distance_per_target.to_frame(user_id).T\n",
+ " target_distance = pd.concat([target_distance, row])\n",
+ " \n",
+ "target_distance.columns = ['distance_' + str(x) for x in target_distance.columns]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "18093734",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "target_duration = df.groupby(['user_id', 'target'])['section_duration_argmax'].sum().unstack()\n",
+ "target_duration.rename(columns=MAP, inplace=True)\n",
+ "target_duration.fillna(0., inplace=True)\n",
+ "target_duration.columns = ['duration_' + str(x) for x in target_duration.columns]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8001a140",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "target_df = user_target_pct.merge(right=target_distance, left_index=True, right_index=True).merge(\n",
+ " right=target_duration, left_index=True, right_index=True\n",
+ ")\n",
+ "\n",
+ "target_df.fillna(0., inplace=True)\n",
+ "\n",
+ "target_df = pd.DataFrame(\n",
+ " MinMaxScaler().fit_transform(target_df),\n",
+ " columns=target_df.columns,\n",
+ " index=target_df.index\n",
+ ")\n",
+ "\n",
+ "display(target_df)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "eba4f246",
+ "metadata": {},
+ "source": [
+ "### Uncomment if you want to find the best eps"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31fecc00",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# epsilons = np.linspace(5e-3, 1., 1500)\n",
+ "# best_score = -np.inf\n",
+ "# best_eps = None\n",
+ "# best_n = None\n",
+ "# # alpha = 0.7\n",
+ "# beta = 0.05\n",
+ "\n",
+ "# for eps in epsilons:\n",
+ "# for n in range(2, 30):\n",
+ "# labels = DBSCAN(eps=eps, min_samples=n).fit(target_df).labels_\n",
+ " \n",
+ "# n_unique = np.unique(labels)\n",
+ "# n_outliers = len(labels[labels == -1])\n",
+ " \n",
+ "# if n_outliers == len(labels) or len(n_unique) < 2:\n",
+ "# continue\n",
+ " \n",
+ "# # Encourage more clustering and discourage more outliers.\n",
+ "# score = silhouette_score(target_df, labels) + (len(labels) - n_outliers)/n_outliers\n",
+ " \n",
+ "# if score > best_score:\n",
+ "# best_score = score\n",
+ "# best_eps = eps\n",
+ "# best_n = n\n",
+ "\n",
+ "# print(f\"{best_score=}, {best_n=}, {best_eps=}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e39b41ba",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 0.35 is a good value\n",
+ "\n",
+ "'''\n",
+ "allCEO = DBSCAN(eps=0.52, min_samples=2)\n",
+ "durham: DBSCAN(eps=best_eps, min_samples=2)\n",
+ "masscec: min_samples=2, eps=0.986724482988659\n",
+ "'''\n",
+ "\n",
+ "cl2 = DBSCAN(eps=0.6, min_samples=2).fit(target_df)\n",
+ "# cl2 = KMeans(n_clusters=5).fit(target_df)\n",
+ "\n",
+ "Counter(cl2.labels_)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1dbf8763",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.decomposition import PCA\n",
+ "\n",
+ "tsfm = PCA(n_components=2).fit_transform(target_df)\n",
+ "\n",
+ "fig, ax = plt.subplots()\n",
+ "sns.scatterplot(x=tsfm[:,0], y=tsfm[:,1], c=cl2.labels_)\n",
+ "ax.set(xlabel='Latent Dim 0', ylabel='Latent Dim 1')\n",
+ "plt.savefig(OUTPUT_DIR / f'{CURRENT_DB}__Fig2__PCA_w_colors.png', dpi=300)\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1e444316",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(df.columns.tolist())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f0bc09b9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Per-cluster users.\n",
+ "from sklearn.preprocessing import OneHotEncoder\n",
+ "from sklearn.preprocessing import MinMaxScaler\n",
+ "from sklearn.ensemble import IsolationForest\n",
+ "from sklearn.svm import OneClassSVM\n",
+ "from sklearn.neighbors import LocalOutlierFactor\n",
+ "from sklearn.tree import DecisionTreeClassifier\n",
+ "\n",
+ "\n",
+ "demographic_cols = {\n",
+ " 'allceo': [ \n",
+ " 'has_drivers_license', 'is_student', 'is_paid', 'income_category', 'n_residence_members', \n",
+ " 'n_residents_u18', 'n_residents_with_license', 'n_motor_vehicles',\n",
+ " 'has_medical_condition', 'ft_job', 'multiple_jobs', 'n_working_residents', \n",
+ " \"highest_education_Bachelor's degree\", 'highest_education_Graduate degree or professional degree', \n",
+ " 'highest_education_High school graduate or GED', 'highest_education_Less than a high school graduate', \n",
+ " 'highest_education_Prefer not to say', 'highest_education_Some college or associates degree', \n",
+ " 'primary_job_description_Clerical or administrative support', 'primary_job_description_Custodial', \n",
+ " 'primary_job_description_Education', \n",
+ " 'primary_job_description_Manufacturing, construction, maintenance, or farming', \n",
+ " 'primary_job_description_Medical/healthcare', 'primary_job_description_Other', 'gender_Man', \n",
+ " 'gender_Man;Nonbinary/genderqueer/genderfluid', 'gender_Nonbinary/genderqueer/genderfluid', \n",
+ " 'gender_Prefer not to say', 'gender_Test', 'gender_Woman', 'gender_Woman;Nonbinary/genderqueer/genderfluid', \n",
+ " 'age_16___20_years_old', 'age_1___5_years_old', 'age_21___25_years_old', 'age_26___30_years_old', \n",
+ " 'age_31___35_years_old', 'age_36___40_years_old', 'age_41___45_years_old', 'age_46___50_years_old', \n",
+ " 'age_51___55_years_old', 'age_56___60_years_old', 'age_61___65_years_old', 'age___65_years_old', \n",
+ " 'av_s_car', 'av_walk', 'av_ridehail', 'av_s_micro', 'av_transit', 'av_no_trip', 'av_car', 'av_unknown', \n",
+ " 'av_p_micro'\n",
+ " ],\n",
+ " 'durham': [\n",
+ " 'is_student', 'is_paid', 'has_drivers_license', 'n_residents_u18', 'n_residence_members', 'income_category',\n",
+ " 'n_residents_with_license', 'n_working_residents', 'n_motor_vehicles', 'has_medical_condition', 'ft_job',\n",
+ " 'multiple_jobs', 'highest_education_bachelor_s_degree', 'highest_education_graduate_degree_or_professional_degree',\n",
+ " 'highest_education_high_school_graduate_or_ged', 'highest_education_less_than_a_high_school_graduate',\n",
+ " 'highest_education_some_college_or_associates_degree', 'primary_job_description_Clerical or administrative support',\n",
+ " 'primary_job_description_Manufacturing, construction, maintenance, or farming',\n",
+ " 'primary_job_description_Other', 'primary_job_description_Prefer not to say', 'primary_job_description_Professional, Manegerial, or Technical',\n",
+ " 'primary_job_description_Sales or service', 'gender_man', 'gender_non_binary_genderqueer_gender_non_confor',\n",
+ " 'gender_prefer_not_to_say', 'gender_woman', 'age_16___20_years_old', 'age_21___25_years_old', 'age_26___30_years_old',\n",
+ " 'age_31___35_years_old', 'age_36___40_years_old', 'age_41___45_years_old', 'age_46___50_years_old',\n",
+ " 'age_51___55_years_old', 'age_56___60_years_old', 'av_unknown', 'av_no_trip', 'av_s_micro', 'av_s_car', 'av_car',\n",
+ " 'av_p_micro', 'av_walk', 'av_transit', 'av_ridehail'\n",
+ " ],\n",
+ " 'nicr': [\n",
+ "\n",
+ " 'is_student', 'is_paid', 'has_drivers_license', 'n_residents_u18', 'n_residence_members', \n",
+ " 'income_category', 'n_residents_with_license', 'n_working_residents', 'n_motor_vehicles', \n",
+ " 'has_medical_condition', 'ft_job', 'multiple_jobs', 'highest_education_bachelor_s_degree', \n",
+ " 'highest_education_high_school_graduate_or_ged', 'highest_education_prefer_not_to_say', \n",
+ " 'highest_education_some_college_or_associates_degree', \n",
+ " 'primary_job_description_Clerical or administrative support', 'primary_job_description_Other', \n",
+ " 'gender_man', 'gender_woman', 'age_16___20_years_old', 'age_21___25_years_old', 'age_26___30_years_old', \n",
+ " 'av_s_car', 'av_no_trip', 'av_s_micro', 'av_walk', 'av_unknown', 'av_p_micro', 'av_transit', 'av_car', \n",
+ " 'av_ridehail'\n",
+ " ],\n",
+ " 'masscec': [\n",
+ " 'is_student', 'is_paid', 'has_drivers_license', 'n_residents_u18', 'n_residence_members', \n",
+ " 'income_category', 'n_residents_with_license', 'n_working_residents', \n",
+ " 'n_motor_vehicles', 'has_medical_condition', 'ft_job', 'multiple_jobs', \n",
+ " 'highest_education_bachelor_s_degree', 'highest_education_graduate_degree_or_professional_degree',\n",
+ " 'highest_education_high_school_graduate_or_ged', \n",
+ " 'highest_education_less_than_a_high_school_graduate', 'highest_education_prefer_not_to_say', \n",
+ " 'highest_education_some_college_or_associates_degree', \n",
+ " 'primary_job_description_Clerical or administrative support', \n",
+ " 'primary_job_description_Manufacturing, construction, maintenance, or farming', \n",
+ " 'primary_job_description_Other', 'primary_job_description_Prefer not to say', \n",
+ " 'primary_job_description_Professional, Manegerial, or Technical', \n",
+ " 'primary_job_description_Sales or service', 'gender_man', \n",
+ " 'gender_non_binary_genderqueer_gender_non_confor', 'gender_prefer_not_to_say', 'gender_woman', \n",
+ " 'age_16___20_years_old', 'age_21___25_years_old', 'age_26___30_years_old', \n",
+ " 'age_31___35_years_old', 'age_36___40_years_old', 'age_41___45_years_old', \n",
+ " 'age_46___50_years_old', 'age_51___55_years_old', 'age_56___60_years_old', \n",
+ " 'age_61___65_years_old', 'age___65_years_old', 'av_no_trip', 'av_transit', \n",
+ " 'av_ridehail', 'av_walk', 'av_car', 'av_p_micro', 'av_unknown', 'av_s_micro', 'av_s_car'\n",
+ " ],\n",
+ " 'ride2own': [\n",
+ " 'has_drivers_license', 'is_student', 'is_paid', 'income_category', 'n_residence_members', \n",
+ " 'n_working_residents', 'n_residents_u18', 'n_residents_with_license', 'n_motor_vehicles', \n",
+ " 'has_medical_condition', 'ft_job', 'multiple_jobs', 'highest_education_bachelor_s_degree', \n",
+ " 'highest_education_graduate_degree_or_professional_degree', \n",
+ " 'highest_education_high_school_graduate_or_ged', \n",
+ " 'highest_education_less_than_a_high_school_graduate', \n",
+ " 'highest_education_some_college_or_associates_degree', 'primary_job_description_Other', \n",
+ " 'primary_job_description_Professional, Manegerial, or Technical', \n",
+ " 'primary_job_description_Sales or service', 'gender_man', \n",
+ " 'gender_non_binary_genderqueer_gender_non_confor', 'gender_woman', 'age_16___20_years_old', \n",
+ " 'age_21___25_years_old', 'age_26___30_years_old', 'age_31___35_years_old', \n",
+ " 'age_36___40_years_old', 'age_41___45_years_old', 'age_51___55_years_old', \n",
+ " 'age_56___60_years_old', 'age___65_years_old', 'av_p_micro', 'av_s_car', 'av_car', \n",
+ " 'av_ridehail', 'av_walk', 'av_transit', 'av_no_trip', 'av_s_micro', 'av_unknown'\n",
+ " ]\n",
+ "}\n",
+ "\n",
+ "\n",
+ "cluster_labels = cl2.labels_\n",
+ "demographics = df.groupby('user_id').first()[demographic_cols[CURRENT_DB]]\n",
+ "demographics = demographics.loc[target_df.index, :]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5a3c6355",
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "### DEMOGRAPHICS\n",
+ "\n",
+ "def entropy(x):\n",
+ " # Compute bincount, normalize over the entire size. Gives us probabilities.\n",
+ " p = np.unique(x, return_counts=True)[1]/len(x)\n",
+ " # Compute the enropy usnig the probabilities.\n",
+ " return -np.sum(p * np.log2(p))\n",
+ "\n",
+ "def preprocess_demo_data(df: pd.DataFrame):\n",
+ " return df\n",
+ "\n",
+ "\n",
+ "within_cluster_homogeneity = dict()\n",
+ "other_cluster_homogeneity = dict()\n",
+ "labels = cl2.labels_\n",
+ "\n",
+ "for cix in np.unique(labels):\n",
+ " within_cluster_homogeneity[cix] = dict()\n",
+ " users = target_df[labels == cix].index\n",
+ " data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ " processed = preprocess_demo_data(data)\n",
+ " \n",
+ " for col in processed.columns:\n",
+ " # Numeric/ordinal values. Use std. to measure homogeneity.\n",
+ " if col in [\n",
+ " 'n_residence_members', 'n_residents_u18', 'n_working_residents', 'n_motor_vehicles',\n",
+ " 'n_residents_with_license', 'income_category'\n",
+ " ]:\n",
+ " within_cluster_homogeneity[cix][col] = processed[col].std()\n",
+ " else:\n",
+ " within_cluster_homogeneity[cix][col] = entropy(processed[col])\n",
+ "\n",
+ "# Compute average homogeneity across other clusters.\n",
+ "for cix in within_cluster_homogeneity.keys():\n",
+ " other_cluster_homogeneity[cix] = dict()\n",
+ " other_clusters = set(within_cluster_homogeneity.keys()) - set([cix])\n",
+ " for feature in within_cluster_homogeneity[cix].keys():\n",
+ " homogeneity_in_others = [within_cluster_homogeneity[x][feature] for x in other_clusters]\n",
+ " other_cluster_homogeneity[cix][feature] = np.mean(homogeneity_in_others)\n",
+ "\n",
+ " \n",
+ "# Compute contrastive homogeneity\n",
+ "# CH = homogeneity within cluster / average homogeneity across other clusters\n",
+ "for cix in within_cluster_homogeneity.keys():\n",
+ " ch_scores = list()\n",
+ " print(f\"For cluster {cix}:\")\n",
+ " for feature in within_cluster_homogeneity[cix].keys():\n",
+ " feature_ch = within_cluster_homogeneity[cix][feature]/(other_cluster_homogeneity[cix][feature] + 1e-6)\n",
+ " ch_scores.append((feature, feature_ch))\n",
+ " \n",
+ " ch_df = pd.DataFrame(ch_scores, columns=['feature', 'ch']).sort_values(by=['ch']).head(4)\n",
+ " \n",
+ " # Display actual values.\n",
+ " users = target_df[labels == cix].index\n",
+ " data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ " processed = preprocess_demo_data(data)\n",
+ " \n",
+ " display(ch_df)\n",
+ " print()\n",
+ " filtered = processed.loc[:, processed.columns.isin(ch_df.feature)][ch_df.feature]\n",
+ " filtered_features = ch_df.feature.tolist()\n",
+ " \n",
+ " fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(12, 10))\n",
+ " for i, a in enumerate(ax.flatten()):\n",
+ " sns.histplot(filtered[filtered_features[i]], ax=a, stat=\"percent\")\n",
+ " plt.tight_layout()\n",
+ " plt.savefig(f\"{CURRENT_DB}_{cix}_Demographic_consistency.png\", dpi=300)\n",
+ " plt.show()\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "580bbd86",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from scipy.stats import iqr\n",
+ "\n",
+ "def get_trip_summary_df(users, df):\n",
+ " '''\n",
+ " Group the trips by user ID and argmax_mode and compute trip summaries. Additional\n",
+ " statistics that could be incorporated: IQR.\n",
+ " \n",
+ " mode_coverage computes trips summaries for the sections with the most-traveled distance.\n",
+ " '''\n",
+ " \n",
+ " costs = [c for c in df.columns if 'av_' in c]\n",
+ " \n",
+ " mode_coverage = df.groupby(['user_id', 'section_mode_argmax'])[\n",
+ " ['section_duration_argmax', 'section_distance_argmax', 'mph'] + costs\n",
+ " ].agg(['mean', 'median']).unstack()\n",
+ " \n",
+ " global_stats = df.groupby('user_id')[['duration', 'distance']].agg(\n",
+ " ['mean', 'median']\n",
+ " )\n",
+ "\n",
+ " mode_coverage.columns = mode_coverage.columns.map('_'.join)\n",
+ " global_stats.columns = global_stats.columns.map('_'.join)\n",
+ " \n",
+ " # return mode_coverage\n",
+ " return mode_coverage.merge(right=global_stats, left_index=True, right_index=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "92ad2485",
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "## TRIP SUMMARIES\n",
+ "\n",
+ "# Per-cluster users.\n",
+ "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
+ "from sklearn.ensemble import IsolationForest\n",
+ "from sklearn.svm import OneClassSVM\n",
+ "from sklearn.neighbors import LocalOutlierFactor\n",
+ "from sklearn.feature_selection import SelectKBest, mutual_info_classif\n",
+ "\n",
+ "labels = cl2.labels_\n",
+ "\n",
+ "def get_data(cix):\n",
+ " users = target_df.iloc[labels == cix, :].index\n",
+ " \n",
+ " # Compute trip summaries.\n",
+ " X = df.loc[df.user_id.isin(users), [\n",
+ " 'section_distance_argmax', 'duration', 'distance', 'section_mode_argmax',\n",
+ " 'section_duration_argmax', 'mph', 'target', 'user_id'\n",
+ " ] + [c for c in df.columns if 'cost_' in c]].reset_index(drop=True)\n",
+ " \n",
+ " # Compute the target distribution and select the argmax.\n",
+ " target_distribution = X.target.value_counts(ascending=False, normalize=True)\n",
+ " target_distribution.rename(index=MAP, inplace=True)\n",
+ " \n",
+ " # Caution - this summary df has NaNs. Use nanstd() to compute nan-aware std.\n",
+ " subset = get_trip_summary_df(users, X)\n",
+ " \n",
+ " norm_subset = pd.DataFrame(\n",
+ " MinMaxScaler().fit_transform(subset),\n",
+ " columns=subset.columns, index=subset.index\n",
+ " )\n",
+ " \n",
+ " return norm_subset, target_distribution\n",
+ "\n",
+ "\n",
+ "in_cluster_homogeneity = dict()\n",
+ "out_cluster_homogeneity = dict()\n",
+ "\n",
+ "for cluster_ix in np.unique(labels):\n",
+ " in_cluster_homogeneity[cluster_ix] = dict()\n",
+ " norm_subset, _ = get_data(cluster_ix)\n",
+ " for feature in norm_subset.columns:\n",
+ " in_cluster_homogeneity[cluster_ix][feature] = np.nanstd(norm_subset[feature])\n",
+ "\n",
+ "for cix in in_cluster_homogeneity.keys():\n",
+ " out_cluster_homogeneity[cix] = dict()\n",
+ " oix = set(labels) - set([cix])\n",
+ " for feature in norm_subset.columns:\n",
+ " out_cluster_homogeneity[cix][feature] = np.nanmean([in_cluster_homogeneity[x].get(feature, np.nan) for x in oix])\n",
+ "\n",
+ "# Now, compute the per-cluster homogeneity.\n",
+ "for cix in in_cluster_homogeneity.keys():\n",
+ " ch = list()\n",
+ " for feature in in_cluster_homogeneity[cix].keys():\n",
+ " if feature in in_cluster_homogeneity[cix] and feature in out_cluster_homogeneity[cix]:\n",
+ " ratio = in_cluster_homogeneity[cix][feature] / (out_cluster_homogeneity[cix][feature] + 1e-6)\n",
+ " ch.append([feature, ratio])\n",
+ " \n",
+ " ch_df = pd.DataFrame(ch, columns=['feature', 'ch']).sort_values(by=['ch']).head(4)\n",
+ " data, target_dist = get_data(cix)\n",
+ " \n",
+ " features = ch_df.feature.tolist()\n",
+ " \n",
+ " print(f\"For cluster {cix}:\")\n",
+ " display(target_dist)\n",
+ " display(ch_df)\n",
+ " \n",
+ " fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(12, 10))\n",
+ " for i, a in enumerate(ax.flatten()):\n",
+ " sns.histplot(data[features[i]], ax=a, stat=\"percent\")\n",
+ " plt.tight_layout()\n",
+ " plt.savefig(f\"{CURRENT_DB}_{cix}_Trip_consistency.png\", dpi=300)\n",
+ " plt.show()\n",
+ " print()\n",
+ " \n",
+ " print(50*'=')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4992ff45",
+ "metadata": {},
+ "source": [
+ "## Now check the combined homogeneity score"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8723e3d",
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "ic, oc = dict(), dict()\n",
+ "\n",
+ "labels = cl2.labels_\n",
+ "TOP_K = 3\n",
+ "\n",
+ "\n",
+ "for cix in np.unique(labels):\n",
+ " ic[cix] = dict()\n",
+ " \n",
+ " # Trip characteristics.\n",
+ " norm_subset, _ = get_data(cix)\n",
+ " for feature in norm_subset.columns:\n",
+ " ic[cix][feature] = np.nanstd(norm_subset[feature])\n",
+ " \n",
+ " # Demographics.\n",
+ " users = target_df[labels == cix].index\n",
+ " data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ " processed = preprocess_demo_data(data)\n",
+ " \n",
+ " for col in processed.columns:\n",
+ " # Numeric/ordinal values. Use std. to measure homogeneity.\n",
+ " if col in [\n",
+ " 'n_residence_members', 'n_residents_u18', 'n_working_residents', 'n_motor_vehicles',\n",
+ " 'n_residents_with_license', 'income_category'\n",
+ " ]:\n",
+ " ic[cix][col] = np.nanstd(processed[col])\n",
+ " else:\n",
+ " ic[cix][col] = entropy(processed[col])\n",
+ "\n",
+ "for cix in ic.keys():\n",
+ " oc[cix] = dict()\n",
+ " oix = set(labels) - set([cix])\n",
+ " for feature in ic[cix].keys():\n",
+ " oc[cix][feature] = np.nanmean([ic[x].get(feature, np.nan) for x in oix])\n",
+ "\n",
+ "per_cluster_most_homogeneous = dict()\n",
+ "\n",
+ "# Now, compute the per-cluster homogeneity.\n",
+ "ax_ix = 0\n",
+ "for cix in ic.keys():\n",
+ "\n",
+ " print(f\"For cluster {cix}:\")\n",
+ "\n",
+ " # For each, cluster, we will have (TOP_K x n_clusters) figures.\n",
+ " fig, ax = plt.subplots(nrows=TOP_K, ncols=len(ic.keys()), figsize=(12, 8))\n",
+ "\n",
+ " other_ix = set(ic.keys()) - set([cix])\n",
+ " \n",
+ " ch = list()\n",
+ " for feature in ic[cix].keys():\n",
+ " if feature in oc[cix]:\n",
+ " ratio = ic[cix][feature] / (oc[cix][feature] + 1e-6)\n",
+ " ch.append([feature, ratio])\n",
+ " \n",
+ " # Just the top k.\n",
+ " ch_df = pd.DataFrame(ch, columns=['feature', 'ch']).sort_values(by=['ch']).reset_index(drop=True).head(TOP_K)\n",
+ "\n",
+ " figure_data = dict()\n",
+ " \n",
+ " # Get the actual trip summary data.\n",
+ " trip_summary_data, target_dist = get_data(cix)\n",
+ " \n",
+ " # Get the actual demographic data.\n",
+ " users = target_df[labels == cix].index\n",
+ " data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ " processed = preprocess_demo_data(data)\n",
+ "\n",
+ " # Left-most subplot will be that of the current cluster's feature.\n",
+ " for row_ix, row in ch_df.iterrows():\n",
+ " if row.feature in trip_summary_data.columns:\n",
+ " sns.histplot(trip_summary_data[row.feature], ax=ax[row_ix][0], stat='percent').set_title(\"Current cluster\")\n",
+ " else:\n",
+ " sns.histplot(processed[row.feature], ax=ax[row_ix][0], stat='percent').set_title(\"Current cluster\")\n",
+ " ax[row_ix][0].set_xlabel(ax[row_ix][0].get_xlabel(), fontsize=8)\n",
+ " ax[row_ix][0].set_ylim(0., 100.)\n",
+ "\n",
+ " offset_col_ix = 1\n",
+ " ## Now, others.\n",
+ " for oix in other_ix:\n",
+ " # Get the actual trip summary data.\n",
+ " other_summary_data, _ = get_data(oix)\n",
+ " \n",
+ " # Get the actual demographic data.\n",
+ " users = target_df[labels == oix].index\n",
+ " data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ " other_demo = preprocess_demo_data(data)\n",
+ "\n",
+ " for row_ix, row in ch_df.iterrows():\n",
+ " if row.feature in other_summary_data.columns:\n",
+ " sns.histplot(other_summary_data[row.feature], ax=ax[row_ix][offset_col_ix], stat='percent').set_title(f\"Cluster {oix}\")\n",
+ " else:\n",
+ " sns.histplot(other_demo[row.feature], ax=ax[row_ix][offset_col_ix], stat='percent').set_title(f\"Cluster {oix}\")\n",
+ " ax[row_ix][offset_col_ix].set_xlabel(ax[row_ix][offset_col_ix].get_xlabel(), fontsize=8)\n",
+ " ax[row_ix][offset_col_ix].set_ylim(0., 100.)\n",
+ " \n",
+ " offset_col_ix += 1\n",
+ " \n",
+ " plt.tight_layout()\n",
+ " plt.savefig(OUTPUT_DIR / f\"{CURRENT_DB}_cluster{cix}_combined_features.png\", dpi=300)\n",
+ " plt.show()\n",
+ " print(50 * '=')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "24a80f68",
+ "metadata": {},
+ "source": [
+ "## Try a different clustering technique? (Unexplored)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d0288db8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# from sklearn.cluster import AffinityPropagation\n",
+ "\n",
+ "# best_score = -np.inf\n",
+ "# best_params = None"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b14ad0c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# cls = AffinityPropagation(random_state=13210).fit(target_df)\n",
+ "# labels = cls.labels_\n",
+ "\n",
+ "# print(labels)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2562bbb6-66eb-4283-8c08-6e20a0b2ade5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# center_embeddings = cls.cluster_centers_\n",
+ "# centers_proj = PCA(n_components=2).fit_transform(center_embeddings)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c7aad38a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# fig, ax = plt.subplots()\n",
+ "# sns.scatterplot(x=tsfm[:,0], y=tsfm[:,1], c=cls.labels_, ax=ax)\n",
+ "# ax.scatter(x=centers_proj[:,0], y=centers_proj[:,1], marker='X', c='red', alpha=0.5)\n",
+ "# ax.set(xlabel='Latent Dim 0', ylabel='Latent Dim 1')\n",
+ "# # plt.legend([str(x) for x in ap_labels], loc='best')\n",
+ "# plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "39ce0238-b3f2-4f46-a52f-13e3160cc52f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# def get_data2(cix, labels):\n",
+ "# users = target_df.iloc[labels == cix, :].index\n",
+ " \n",
+ "# # Compute trip summaries.\n",
+ "# X = df.loc[df.user_id.isin(users), [\n",
+ "# 'section_distance_argmax', 'section_duration_argmax',\n",
+ "# 'section_mode_argmax', 'distance',\n",
+ "# 'duration', 'mph', 'user_id', 'target'\n",
+ "# ]]\n",
+ " \n",
+ "# # Compute the target distribution and select the argmax.\n",
+ "# target_distribution = X.target.value_counts(ascending=False, normalize=True)\n",
+ "# target_distribution.rename(index=MAP, inplace=True)\n",
+ " \n",
+ "# # Caution - this summary df has NaNs. Use nanstd() to compute nan-aware std.\n",
+ "# subset = get_trip_summary_df(users, X)\n",
+ " \n",
+ "# norm_subset = pd.DataFrame(\n",
+ "# MinMaxScaler().fit_transform(subset),\n",
+ "# columns=subset.columns, index=subset.index\n",
+ "# )\n",
+ " \n",
+ "# return norm_subset, target_distribution"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ec27cf29",
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "# ## Analaysis for this data.\n",
+ "\n",
+ "# ic, oc = dict(), dict()\n",
+ "# labels = cls.labels_\n",
+ "\n",
+ "# for cix in np.unique(labels):\n",
+ "# users = target_df[labels == cix].index\n",
+ " \n",
+ "# ic[cix] = dict()\n",
+ " \n",
+ "# # Trip characteristics.\n",
+ "# norm_subset, _ = get_data2(cix, labels)\n",
+ "# for feature in norm_subset.columns:\n",
+ "# ic[cix][feature] = np.nanstd(norm_subset[feature])\n",
+ " \n",
+ "# # Demographics.\n",
+ "# data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ "# processed = preprocess_demo_data(data)\n",
+ " \n",
+ "# for col in processed.columns:\n",
+ "# # Numeric/ordinal values. Use std. to measure homogeneity.\n",
+ "# if col == 'age' or col == 'income_category' or col == 'n_working_residents':\n",
+ "# ic[cix][col] = np.nanstd(processed[col])\n",
+ "# else:\n",
+ "# ic[cix][col] = entropy(processed[col])\n",
+ "\n",
+ "# for cix in ic.keys():\n",
+ "# oc[cix] = dict()\n",
+ "# oix = set(labels) - set([cix])\n",
+ "# for feature in ic[cix].keys():\n",
+ "# oc[cix][feature] = np.nanmean([ic[x].get(feature, np.nan) for x in oix])\n",
+ "\n",
+ "# # # Now, compute the per-cluster homogeneity.\n",
+ "# # for cix in ic.keys():\n",
+ " \n",
+ "# # users = users = target_df[labels == cix].index\n",
+ "# # norm_subset, target_dist = get_data(cix, labels)\n",
+ "# # data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ "# # processed = preprocess_demo_data(data)\n",
+ " \n",
+ "# # concat = processed.merge(norm_subset, left_index=True, right_index=True)\n",
+ " \n",
+ "# # ch = list()\n",
+ "# # for feature in ic[cix].keys():\n",
+ "# # ratio = ic[cix][feature] / (oc[cix][feature] + 1e-6)\n",
+ "# # ch.append([feature, ratio])\n",
+ " \n",
+ "# # ch_df = pd.DataFrame(ch, columns=['feature', 'ch']).sort_values(by=['ch']).head(TOP_K).reset_index(drop=True)\n",
+ "\n",
+ "\n",
+ "# # Now, compute the per-cluster homogeneity.\n",
+ "# ax_ix = 0\n",
+ "# for cix in ic.keys():\n",
+ "\n",
+ "# print(f\"For cluster {cix}:\")\n",
+ "\n",
+ "# # For each, cluster, we will have (TOP_K x n_clusters) figures.\n",
+ "# fig, ax = plt.subplots(nrows=5, ncols=len(ic.keys()), figsize=(12, 8))\n",
+ "\n",
+ "# other_ix = set(ic.keys()) - set([cix])\n",
+ " \n",
+ "# ch = list()\n",
+ "# for feature in ic[cix].keys():\n",
+ "# ratio = ic[cix][feature] / (oc[cix][feature] + 1e-6)\n",
+ "# ch.append([feature, ratio])\n",
+ " \n",
+ "# # Just the top k.\n",
+ "# ch_df = pd.DataFrame(ch, columns=['feature', 'ch']).sort_values(by=['ch']).reset_index(drop=True).head(5)\n",
+ "# figure_data = dict()\n",
+ " \n",
+ "# # Get the actual trip summary data.\n",
+ "# trip_summary_data, target_dist = get_data(cix)\n",
+ "\n",
+ "# display(target_dist)\n",
+ " \n",
+ "# # Get the actual demographic data.\n",
+ "# users = target_df[labels == cix].index\n",
+ "# data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ "# processed = preprocess_demo_data(data)\n",
+ "\n",
+ "# # Left-most subplot will be that of the current cluster's feature.\n",
+ "# for row_ix, row in ch_df.iterrows():\n",
+ "# if row.feature in trip_summary_data.columns:\n",
+ "# sns.histplot(trip_summary_data[row.feature], ax=ax[row_ix][0], stat='percent').set_title(\"Current cluster\")\n",
+ "# else:\n",
+ "# sns.histplot(processed[row.feature], ax=ax[row_ix][0], stat='percent').set_title(\"Current cluster\")\n",
+ "# ax[row_ix][0].set_xlabel(ax[row_ix][0].get_xlabel(), fontsize=6)\n",
+ "# ax[row_ix][0].set_ylim(0., 100.)\n",
+ "\n",
+ "# offset_col_ix = 1\n",
+ "# ## Now, others.\n",
+ "# for oix in other_ix:\n",
+ "# # Get the actual trip summary data.\n",
+ "# other_summary_data, _ = get_data(oix)\n",
+ " \n",
+ "# # Get the actual demographic data.\n",
+ "# users = target_df[labels == oix].index\n",
+ "# data = demographics.loc[demographics.index.isin(users), :].reset_index(drop=True)\n",
+ "# other_demo = preprocess_demo_data(data)\n",
+ "\n",
+ "# for row_ix, row in ch_df.iterrows():\n",
+ "# if row.feature in other_summary_data.columns:\n",
+ "# sns.histplot(other_summary_data[row.feature], ax=ax[row_ix][offset_col_ix], stat='percent').set_title(f\"Cluster {oix}\")\n",
+ "# else:\n",
+ "# sns.histplot(other_demo[row.feature], ax=ax[row_ix][offset_col_ix], stat='percent').set_title(f\"Cluster {oix}\")\n",
+ "# ax[row_ix][offset_col_ix].set_xlabel(ax[row_ix][offset_col_ix].get_xlabel(), fontsize=6)\n",
+ "# ax[row_ix][offset_col_ix].set_ylim(0., 100.)\n",
+ " \n",
+ "# offset_col_ix += 1\n",
+ "\n",
+ "# plt.tight_layout()\n",
+ "# plt.show()\n",
+ "# print(50 * '=')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c0b642db",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "emission",
+ "language": "python",
+ "name": "emission"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.16"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/replacement_mode_modeling/05_biogeme_modeling.ipynb b/replacement_mode_modeling/05_biogeme_modeling.ipynb
new file mode 100644
index 00000000..29e89b76
--- /dev/null
+++ b/replacement_mode_modeling/05_biogeme_modeling.ipynb
@@ -0,0 +1,929 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Install biogeme: `pip3 install biogeme==3.2.12`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "from enum import Enum\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "\n",
+ "import pandas as pd\n",
+ "import biogeme.biogeme as bio\n",
+ "import biogeme.database as db\n",
+ "from biogeme import models\n",
+ "from biogeme.expressions import Beta, DefineVariable\n",
+ "from biogeme.expressions import Variable\n",
+ "import numpy as np\n",
+ "import seaborn as sns\n",
+ "\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "from sklearn.linear_model import LinearRegression\n",
+ "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
+ "from sklearn.metrics import f1_score, r2_score, ConfusionMatrixDisplay\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Global experiment flags and variables.\n",
+ "SEED = 19348\n",
+ "TARGETS = ['p_micro', 'no_trip', 's_car', 'transit', 'car', 's_micro', 'ridehail', 'walk', 'unknown']\n",
+ "\n",
+ "# Set the Numpy seed too.\n",
+ "np.random.seed(SEED)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SPLIT_TYPE(Enum):\n",
+ " INTRA_USER = 0\n",
+ " TARGET = 1\n",
+ " MODE = 2\n",
+ " \n",
+ "\n",
+ "class SPLIT(Enum):\n",
+ " TRAIN = 0\n",
+ " TEST = 1\n",
+ "\n",
+ "\n",
+ "def get_train_test_splits(data: pd.DataFrame, how=SPLIT_TYPE, test_ratio=0.2, shuffle=True):\n",
+ "\n",
+ " if how == SPLIT_TYPE.INTRA_USER:\n",
+ " \n",
+ " # There are certain users with only one observation. What do we do with those?\n",
+ " # As per the mobilitynet modeling pipeline, we randomly assign them to either the\n",
+ " # training or test set.\n",
+ " \n",
+ " value_counts = data.user_id.value_counts()\n",
+ " single_count_ids = value_counts[value_counts == 1].index\n",
+ " \n",
+ " data_filtered = data.loc[~data.user_id.isin(single_count_ids), :].reset_index(drop=True)\n",
+ " data_single_counts = data.loc[data.user_id.isin(single_count_ids), :].reset_index(drop=True)\n",
+ " \n",
+ " X_tr, X_te = train_test_split(\n",
+ " data_filtered, test_size=test_ratio, shuffle=shuffle, stratify=data_filtered.user_id,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " data_single_counts['assigned'] = np.random.choice(['train', 'test'], len(data_single_counts))\n",
+ " X_tr_merged = pd.concat(\n",
+ " [X_tr, data_single_counts.loc[data_single_counts.assigned == 'train', :].drop(\n",
+ " columns=['assigned'], inplace=False\n",
+ " )],\n",
+ " ignore_index=True, axis=0\n",
+ " )\n",
+ " \n",
+ " X_te_merged = pd.concat(\n",
+ " [X_te, data_single_counts.loc[data_single_counts.assigned == 'test', :].drop(\n",
+ " columns=['assigned'], inplace=False\n",
+ " )],\n",
+ " ignore_index=True, axis=0\n",
+ " )\n",
+ " \n",
+ " return X_tr_merged, X_te_merged\n",
+ " \n",
+ " elif how == SPLIT_TYPE.TARGET:\n",
+ " \n",
+ " X_tr, X_te = train_test_split(\n",
+ " data, test_size=test_ratio, shuffle=shuffle, stratify=data.target,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " return X_tr, X_te\n",
+ " \n",
+ " elif how == SPLIT_TYPE.MODE:\n",
+ " \n",
+ " X_tr, X_te = train_test_split(\n",
+ " data, test_size=test_ratio, shuffle=shuffle, stratify=data.section_mode_argmax,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " return X_tr, X_te\n",
+ " \n",
+ " raise NotImplementedError(\"Unknown split type\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Modeling"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The following are common features across all datasets:\n",
+ "\n",
+ "```\n",
+ "{'age_21___25_years_old', 'cost_unknown', 'start_local_dt_hour', 'av_walk', 'distance', 'duration', 'av_unknown', 'ft_job', 'end_local_dt_hour', 'cost_no_trip', 'cost_s_micro', 'mph', 'n_residents_u18', 'is_paid', 'n_motor_vehicles', 'target', 'n_working_residents', 'section_distance_argmax', 'n_residence_members', 'has_medical_condition', 'primary_job_description_Other', 'cost_walk', 'cost_p_micro', 'av_transit', 'age_16___20_years_old', 'income_category', 'av_s_car', 'av_no_trip', 'cost_s_car', 'multiple_jobs', 'n_residents_with_license', 'section_duration_argmax', 'age_26___30_years_old', 'cost_car', 'av_p_micro', 'av_ridehail', 'av_car', 'cost_transit', 'available_modes', 'av_s_micro', 'has_drivers_license', 'cost_ridehail', 'user_id', 'section_mode_argmax', 'is_student'}\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read the data.\n",
+ "\n",
+ "DATA_SOURCES = [\n",
+ " ('../data/filtered_data/preprocessed_data_Stage_database.csv', 'allceo'),\n",
+ " ('../data/filtered_data/preprocessed_data_openpath_prod_uprm_nicr.csv', 'nicr'),\n",
+ " ('../data/filtered_data/preprocessed_data_openpath_prod_durham.csv', 'durham')\n",
+ "]\n",
+ "\n",
+ "DB_IX = 2\n",
+ "\n",
+ "PATH = DATA_SOURCES[DB_IX][0]\n",
+ "CURRENT_DB = DATA_SOURCES[DB_IX][1]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data = pd.read_csv(PATH)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data.drop_duplicates(inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(data.shape)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def norm_data(df: pd.DataFrame, split: SPLIT, scaler=None):\n",
+ " \n",
+ " columns = df.columns.tolist()\n",
+ " \n",
+ " # Ignore dummy features (1/0).\n",
+ " ignore_cols = [\n",
+ " c for c in columns if 'age_' in c or 'av_' in c or 'gender_' in c \n",
+ " or 'primary_job_description' in c or 'is_' in c or 'highest_education' in c\n",
+ " or '_job' in c or 'has_' in c\n",
+ " ] + ['user_id', 'target', 'section_mode_argmax']\n",
+ " \n",
+ " data = df.loc[:, [c for c in df.columns if c not in ignore_cols]]\n",
+ " ignored = df.loc[:, ignore_cols]\n",
+ " \n",
+ " if split == SPLIT.TRAIN:\n",
+ " \n",
+ " scaler = StandardScaler()\n",
+ " \n",
+ " scaled = pd.DataFrame(\n",
+ " scaler.fit_transform(data), \n",
+ " columns=data.columns, \n",
+ " index=data.index\n",
+ " )\n",
+ " \n",
+ " elif split == SPLIT.TEST:\n",
+ " scaled = pd.DataFrame(\n",
+ " scaler.transform(data), \n",
+ " columns=data.columns, \n",
+ " index=data.index\n",
+ " )\n",
+ " \n",
+ " else:\n",
+ " raise NotImplementedError(\"Unknown split\")\n",
+ " \n",
+ " return pd.concat([scaled, ignored], axis=1), scaler"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def drop_columns(df: pd.DataFrame):\n",
+ " \n",
+ " to_drop = [\n",
+ " 'available_modes'\n",
+ " ]\n",
+ " \n",
+ " for col in to_drop:\n",
+ " if col in df.columns:\n",
+ " df.drop(columns=[col], inplace=True)\n",
+ " \n",
+ " return df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_duration_estimate(df: pd.DataFrame, dset: SPLIT, model_dict: dict):\n",
+ " \n",
+ " X_features = ['section_distance_argmax', 'mph']\n",
+ " \n",
+ " if dset == SPLIT.TRAIN and model_dict is None:\n",
+ " model_dict = dict()\n",
+ " \n",
+ " if dset == SPLIT.TEST and model_dict is None:\n",
+ " raise AttributeError(\"Expected model dict for testing.\")\n",
+ " \n",
+ " if dset == SPLIT.TRAIN:\n",
+ " for section_mode in df.section_mode_argmax.unique():\n",
+ " section_data = df.loc[df.section_mode_argmax == section_mode, :]\n",
+ " if section_mode not in model_dict:\n",
+ " model_dict[section_mode] = dict()\n",
+ "\n",
+ " model = LinearRegression(fit_intercept=True)\n",
+ "\n",
+ " X = section_data[\n",
+ " X_features\n",
+ " ]\n",
+ " Y = section_data[['section_duration_argmax']]\n",
+ "\n",
+ " model.fit(X, Y.values.ravel())\n",
+ "\n",
+ " r2 = r2_score(y_pred=model.predict(X), y_true=Y.values.ravel())\n",
+ " print(f\"Train R2 for {section_mode}: {r2}\")\n",
+ "\n",
+ " model_dict[section_mode]['model'] = model\n",
+ " \n",
+ " elif dset == SPLIT.TEST:\n",
+ " for section_mode in df.section_mode_argmax.unique():\n",
+ " \n",
+ " section_data = df.loc[df.section_mode_argmax == section_mode, :]\n",
+ " \n",
+ " X = section_data[\n",
+ " X_features\n",
+ " ]\n",
+ " Y = section_data[['section_duration_argmax']]\n",
+ " \n",
+ " if section_mode not in model_dict:\n",
+ " y_pred = [np.nan for _ in range(len(X))]\n",
+ " else:\n",
+ " y_pred = model_dict[section_mode]['model'].predict(X)\n",
+ " \n",
+ " r2 = r2_score(y_pred=y_pred, y_true=Y.values.ravel())\n",
+ " print(f\"Test R2 for {section_mode}: {r2}\")\n",
+ " \n",
+ " # Create the new columns for the duration.\n",
+ " new_columns = ['p_micro','no_trip','s_car','transit','car','s_micro','ridehail','walk','unknown']\n",
+ " df[new_columns] = 0\n",
+ " df['temp'] = 0\n",
+ " \n",
+ " for section in df.section_mode_argmax.unique():\n",
+ " X_section = df.loc[df.section_mode_argmax == section, X_features]\n",
+ " \n",
+ " # broadcast to all columns.\n",
+ " df.loc[df.section_mode_argmax == section, 'temp'] = model_dict[section]['model'].predict(X_section)\n",
+ " \n",
+ " for c in new_columns:\n",
+ " df[c] = df['av_' + c] * df['temp']\n",
+ " \n",
+ " df.drop(columns=['temp'], inplace=True)\n",
+ " \n",
+ " df.rename(columns=dict([(x, 'tt_'+x) for x in new_columns]), inplace=True)\n",
+ " \n",
+ " # return model_dict, result_df\n",
+ " return model_dict, df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Now, we drop columns, split the data, and normalize\n",
+ "\n",
+ "data = drop_columns(data)\n",
+ "\n",
+ "train_data, test_data = get_train_test_splits(data=data, how=SPLIT_TYPE.INTRA_USER, shuffle=True)\n",
+ "\n",
+ "train_data, scaler = norm_data(train_data, split=SPLIT.TRAIN)\n",
+ "test_data, _ = norm_data(test_data, SPLIT.TEST, scaler)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "USERS = list(data.user_id.unique())\n",
+ "\n",
+ "USER_MAP = {\n",
+ " u: i+1 for (i, u) in enumerate(USERS)\n",
+ "}\n",
+ "\n",
+ "train_data['user_id'] = train_data['user_id'].apply(lambda x: USER_MAP[x])\n",
+ "test_data['user_id'] = test_data['user_id'].apply(lambda x: USER_MAP[x])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(7, 7))\n",
+ "train_data.target.hist(ax=ax[0])\n",
+ "test_data.target.hist(ax=ax[1])\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "params, train_data = get_duration_estimate(train_data, SPLIT.TRAIN, None)\n",
+ "print(10 * \"-\")\n",
+ "_, test_data = get_duration_estimate(test_data, SPLIT.TEST, params)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Drop section_mode\n",
+ "\n",
+ "train_data.drop(columns=['section_mode_argmax'], inplace=True)\n",
+ "# test_data.drop(columns=['section_mode_argmax'], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_data.shape, test_data.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(train_data.columns.tolist())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Some helper functions that will help ease redundancy in the code.\n",
+ "\n",
+ "def get_database(df: pd.DataFrame, split: SPLIT):\n",
+ " return db.Database(split.name + '_db', df)\n",
+ "\n",
+ "\n",
+ "def get_variables(database: db.Database):\n",
+ " \n",
+ " columns = database.data\n",
+ " \n",
+ " # User-level features.\n",
+ " START_HOUR = Variable('start_local_dt_hour')\n",
+ " END_HOUR = Variable('end_local_dt_hour')\n",
+ " TRIP_DISTANCE = Variable('distance')\n",
+ " INCOME = Variable('income_category')\n",
+ " N_MEMBERS = Variable('n_residence_members')\n",
+ " N_U18 = Variable('n_residents_u18')\n",
+ " N_LICENSE = Variable('n_residents_with_license')\n",
+ " N_VEHICLES = Variable('n_motor_vehicles')\n",
+ " LICENSE = Variable('has_drivers_license')\n",
+ " CONDITION = Variable('has_medical_condition')\n",
+ " FT_JOB = Variable('ft_job')\n",
+ " MULTIPLE_JOBS = Variable('multiple_jobs')\n",
+ " \n",
+ " # Sections\n",
+ " DISTANCE_ARGMAX = Variable('section_distance_argmax')\n",
+ " TT_ARGMAX = Variable('section_duration_argmax')\n",
+ " MPH = Variable('mph')\n",
+ " \n",
+ " # Costs\n",
+ " COST_P_MICRO = Variable('cost_p_micro')\n",
+ " COST_NO_TRIP = Variable('cost_no_trip')\n",
+ " COST_S_CAR = Variable('cost_s_car')\n",
+ " COST_CAR = Variable('cost_car')\n",
+ " COST_S_MICRO = Variable('cost_s_micro')\n",
+ " COST_RIDEHAIL = Variable('cost_ridehail')\n",
+ " COST_WALK = Variable('cost_walk')\n",
+ " COST_UNKNOWN = Variable('cost_unknown')\n",
+ " COST_TRANSIT = Variable('cost_transit')\n",
+ "\n",
+ " # Availability.\n",
+ " AV_P_MICRO = Variable('av_p_micro')\n",
+ " AV_NO_TRIP = Variable('av_no_trip')\n",
+ " AV_S_CAR = Variable('av_s_car')\n",
+ " AV_TRANSIT = Variable('av_transit')\n",
+ " AV_CAR = Variable('av_car')\n",
+ " AV_S_MICRO = Variable('av_s_micro')\n",
+ " AV_RIDEHAIL = Variable('av_ridehail')\n",
+ " AV_WALK = Variable('av_walk')\n",
+ " AV_UNKNOWN = Variable('av_unknown')\n",
+ " \n",
+ " # OHE\n",
+ " G = [Variable(x) for x in columns if 'gender_' in x]\n",
+ " E = [Variable(x) for x in columns if 'highest_education' in x]\n",
+ " PJ = [Variable(x) for x in columns if 'primary_job_description' in x]\n",
+ " \n",
+ " # Times.\n",
+ " TT_P_MICRO = Variable('tt_p_micro')\n",
+ " TT_NO_TRIP = Variable('tt_no_trip')\n",
+ " TT_S_CAR = Variable('tt_s_car')\n",
+ " TT_TRANSIT = Variable('tt_transit')\n",
+ " TT_CAR = Variable('tt_car')\n",
+ " TT_S_MICRO = Variable('tt_s_micro')\n",
+ " TT_RIDEHAIL = Variable('tt_ridehail')\n",
+ " TT_WALK = Variable('tt_walk')\n",
+ " TT_UNKNOWN = Variable('tt_unknown')\n",
+ " \n",
+ " # Choice.\n",
+ " CHOICE = Variable('target')\n",
+ " \n",
+ " return_dict = locals().copy()\n",
+ " \n",
+ " # Remove the gender list and place them in the locals dict.\n",
+ " for i, val in enumerate(G):\n",
+ " return_dict.update({'G_' + str(i): val})\n",
+ " \n",
+ " del return_dict['G']\n",
+ " \n",
+ " \n",
+ " ## Education\n",
+ " for i, val in enumerate(E):\n",
+ " return_dict.update({'E_' + str(i): val})\n",
+ " \n",
+ " del return_dict['E']\n",
+ " \n",
+ " ## Job\n",
+ " for i, val in enumerate(PJ):\n",
+ " return_dict.update({'PJ_' + str(i): val})\n",
+ " \n",
+ " del return_dict['PJ']\n",
+ " \n",
+ " # return the filtered locals() dictionary.\n",
+ " return {k:v for k,v in return_dict.items() if not k.startswith('_') and k not in ['database', 'columns']}\n",
+ "\n",
+ "\n",
+ "# def exclude_from_db(v_dict: dict, db: db.Database):\n",
+ "# EXCLUDE = (v_dict['CHOICE'] == 2) + (v_dict['CHOICE'] == 9) > 0\n",
+ "# db.remove(EXCLUDE)\n",
+ "\n",
+ "def get_params(variables):\n",
+ " \n",
+ " param_dict = {'B_' + k: Beta('B_' + k, 0, None, None, 0) for k in variables.keys()}\n",
+ " \n",
+ " param_dict['ASC_P_MICRO'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " param_dict['ASC_NO_TRIP'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " param_dict['ASC_S_CAR'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " param_dict['ASC_TRANSIT'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " param_dict['ASC_CAR'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " param_dict['ASC_S_MICRO'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " param_dict['ASC_RIDEHAIL'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " param_dict['ASC_WALK'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " param_dict['ASC_UNKNOWN'] = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " \n",
+ " # Return filtered locals dict.\n",
+ " return param_dict\n",
+ "\n",
+ "\n",
+ "def get_utility_functions(v: dict):\n",
+ " \n",
+ " ## User-level utility.\n",
+ " user = 1.\n",
+ " for var in [\n",
+ " 'INCOME', 'N_MEMBERS', \n",
+ " 'N_U18', 'N_LICENSE', 'N_VEHICLES', 'LICENSE', 'CONDITION', 'FT_JOB', 'MULTIPLE_JOBS'\n",
+ " ]:\n",
+ " user += v[var] * v['B_'+var]\n",
+ " \n",
+ " # OHE (One-hot encoded utility.)\n",
+ " ohe = 1.\n",
+ " ohe_vars = [var for var in v if ('G_' in var or 'E_' in var or 'PJ_' in var) and 'B_' not in var]\n",
+ " for var in ohe_vars:\n",
+ " ohe += v[var] * v['B_'+var]\n",
+ " \n",
+ " ## Trip utility.\n",
+ " trip = 1.\n",
+ " for var in ['MPH', 'DISTANCE_ARGMAX', 'TT_ARGMAX', 'START_HOUR', 'END_HOUR', 'TRIP_DISTANCE']:\n",
+ " trip += v[var] * v['B_' + var]\n",
+ " \n",
+ " \n",
+ " V_P_MICRO = v['ASC_P_MICRO'] + \\\n",
+ " ohe + user + trip + \\\n",
+ " v['TT_P_MICRO'] * v['B_TT_P_MICRO'] + \\\n",
+ " v['COST_P_MICRO'] * v['B_COST_P_MICRO']\n",
+ " \n",
+ " V_S_MICRO = v['ASC_S_MICRO'] + \\\n",
+ " ohe + user + trip + \\\n",
+ " v['TT_S_MICRO'] * v['B_TT_S_MICRO'] + \\\n",
+ " v['COST_S_MICRO'] * v['B_COST_S_MICRO']\n",
+ " \n",
+ " V_S_CAR = v['ASC_S_CAR'] + \\\n",
+ " ohe + user + trip + \\\n",
+ " v['TT_S_CAR'] * v['B_TT_S_CAR'] + \\\n",
+ " v['COST_S_CAR'] * v['B_COST_S_CAR']\n",
+ " \n",
+ " V_CAR = v['ASC_CAR'] + \\\n",
+ " ohe + user + trip + \\\n",
+ " v['TT_CAR'] * v['B_TT_CAR'] + \\\n",
+ " v['COST_CAR'] * v['B_COST_CAR']\n",
+ " \n",
+ " V_TRANSIT = v['ASC_TRANSIT'] + \\\n",
+ " ohe + user + trip + \\\n",
+ " v['TT_TRANSIT'] * v['B_TT_TRANSIT'] + \\\n",
+ " v['COST_TRANSIT'] * v['B_COST_TRANSIT']\n",
+ " \n",
+ " V_WALK = v['ASC_WALK'] + \\\n",
+ " ohe + user + trip + \\\n",
+ " v['TT_WALK'] * v['B_TT_WALK'] + \\\n",
+ " v['COST_WALK'] * v['B_COST_WALK']\n",
+ " \n",
+ " V_RIDEHAIL = v['ASC_RIDEHAIL'] + \\\n",
+ " ohe + user + trip + \\\n",
+ " v['TT_RIDEHAIL'] * v['B_TT_RIDEHAIL'] + \\\n",
+ " v['COST_RIDEHAIL'] * v['B_COST_RIDEHAIL']\n",
+ " \n",
+ " V_NO_TRIP = -100\n",
+ " V_UNKNOWN = -100\n",
+ " \n",
+ " # Remember to exclude the input argument.\n",
+ " return {k:v for k,v in locals().items() if not k.startswith('_') and k != 'v'}\n",
+ "\n",
+ "\n",
+ "def get_utility_mapping(var: dict):\n",
+ " # Map alterative to utility functions.\n",
+ " return {\n",
+ " 1: var['V_P_MICRO'], \n",
+ " 2: var['V_NO_TRIP'],\n",
+ " 3: var['V_S_CAR'], \n",
+ " 4: var['V_TRANSIT'],\n",
+ " 5: var['V_CAR'], \n",
+ " 6: var['V_S_MICRO'],\n",
+ " 7: var['V_RIDEHAIL'], \n",
+ " 8: var['V_WALK'], \n",
+ " 9: var['V_UNKNOWN']\n",
+ " }\n",
+ "\n",
+ "\n",
+ "def get_availability_mapping(var: dict):\n",
+ " return {\n",
+ " 1: var['AV_P_MICRO'],\n",
+ " 2: var['AV_NO_TRIP'],\n",
+ " 3: var['AV_S_CAR'],\n",
+ " 4: var['AV_TRANSIT'],\n",
+ " 5: var['AV_CAR'],\n",
+ " 6: var['AV_S_MICRO'],\n",
+ " 7: var['AV_RIDEHAIL'],\n",
+ " 8: var['AV_WALK'],\n",
+ " 9: var['AV_UNKNOWN']\n",
+ " }"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # First, drop columns.\n",
+ "\n",
+ "# train_data = drop_columns(train_data)\n",
+ "\n",
+ "# train_data, scaler = norm_data(train_data, split=SPLIT.TRAIN)\n",
+ "\n",
+ "# get dbs.\n",
+ "train_db = get_database(train_data, SPLIT.TRAIN)\n",
+ "\n",
+ "# get vars.\n",
+ "train_vars = get_variables(train_db)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_vars"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_params = get_params(train_vars)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_params"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_vars.update(train_params)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_V = get_utility_functions(train_vars)\n",
+ "train_vars.update(train_V)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "V = get_utility_mapping(train_vars)\n",
+ "av = get_availability_mapping(train_vars)\n",
+ "logprob = models.loglogit(V, av, train_vars['CHOICE'])\n",
+ "\n",
+ "# logit1 = models.logit(V, av, 1)\n",
+ "# logit2 = models.logit(V, av, 2)\n",
+ "# logit3 = models.logit(V, av, 3)\n",
+ "# logit4 = models.logit(V, av, 4)\n",
+ "# logit5 = models.logit(V, av, 5)\n",
+ "# logit6 = models.logit(V, av, 6)\n",
+ "# logit7 = models.logit(V, av, 7)\n",
+ "# logit8 = models.logit(V, av, 8)\n",
+ "# logit9 = models.logit(V, av, 9)\n",
+ "\n",
+ "# models = {f'logit_{ix}': logit for ix, logit in enumerate(\n",
+ "# [logit1, logit2, logit3, logit4, logit5, logit6, logit7, logit8, logit9]\n",
+ "# )}\n",
+ "\n",
+ "model = bio.BIOGEME(train_db, logprob)\n",
+ "model.modelName = 'customUtility-new'\n",
+ "model.generate_html = False\n",
+ "model.generate_pickle = False"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_results = model.estimate()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(train_results.short_summary())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(train_results.getEstimatedParameters())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from biogeme.expressions import Derive\n",
+ "\n",
+ "\n",
+ "def simulate_results(V, av, db, beta_dict):\n",
+ " \n",
+ " wtp = {\n",
+ " 'WTP s_car': Derive(V[3], 'tt_s_car')/Derive(V[3], 'scaled_cost_s_car'),\n",
+ " 'WTP transit': Derive(V[4], 'tt_transit')/Derive(V[4], 'scaled_cost_transit'),\n",
+ " 'WTP car': Derive(V[5], 'tt_car')/Derive(V[5], 'scaled_cost_car'),\n",
+ " 'WTP s_micro': Derive(V[6], 'tt_s_micro')/Derive(V[6], 'scaled_cost_s_micro'),\n",
+ " 'WTP ridehail': Derive(V[7], 'tt_ridehail')/Derive(V[7], 'scaled_cost_ridehail')\n",
+ " }\n",
+ " \n",
+ " prob_labels = ['Prob. ' + x for x in TARGETS]\n",
+ " probs = [models.logit(V, av, i+1) for i in range(len(prob_labels))]\n",
+ " \n",
+ " simulate = dict(zip(prob_labels, probs))\n",
+ " \n",
+ " # simulate.update(wtp)\n",
+ " \n",
+ " biosim = bio.BIOGEME(db, simulate)\n",
+ " biosim.modelName = 'test-3'\n",
+ " \n",
+ " return biosim.simulate(theBetaValues=beta_dict)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_data = drop_columns(test_data)\n",
+ "\n",
+ "# Scale cost.\n",
+ "test_data, _ = norm_data(test_data, SPLIT.TEST, scaler)\n",
+ "\n",
+ "test_data.drop(columns=['section_mode_argmax'], inplace=True)\n",
+ "\n",
+ "# get dbs.\n",
+ "test_db = get_database(test_data, SPLIT.TEST)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_probs = simulate_results(V, av, test_db, train_results.getBetaValues())\n",
+ "# test_utilities = get_utility_df(train_results, test_data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(test_probs.head())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# argmax starts from 0. Offset all predicted indices by 1.\n",
+ "choices = np.argmax(test_probs.values, axis=1) + 1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "y_true = test_data.chosen\n",
+ "score = f1_score(y_true, choices, average='weighted')\n",
+ "\n",
+ "print(score)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, ax = plt.subplots()\n",
+ "counts = pd.Series(choices).value_counts()\n",
+ "ix = counts.index.tolist()\n",
+ "_x = [i+1 for i in range(len(TARGETS))]\n",
+ "height = [0 if i not in ix else counts[i] for i in _x]\n",
+ "ax.bar(x=_x, height=height)\n",
+ "ax.set_xticks(range(1, 10, 1))\n",
+ "ax.set_xticklabels(TARGETS, rotation=45)\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.metrics import ConfusionMatrixDisplay\n",
+ "\n",
+ "fig, ax = plt.subplots()\n",
+ "cm = ConfusionMatrixDisplay.from_predictions(y_true=y_true, y_pred=choices, ax=ax)\n",
+ "\n",
+ "y_unique = np.unique(y_true)\n",
+ "labelset = [t for i, t in enumerate(TARGETS) if (i+1) in y_unique]\n",
+ "\n",
+ "ax.set_xticklabels(labelset, rotation=45)\n",
+ "ax.set_yticklabels(labelset)\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# np.diag(cm.confusion_matrix)/np.sum(cm.confusion_matrix, axis=1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# u_np = test_utilities.values\n",
+ "# choice_df = np.exp(u_np)/np.sum(np.exp(u_np), axis=1, keepdims=True)\n",
+ "\n",
+ "# choice_df = pd.DataFrame(choice_df, columns=test_utilities.columns)\n",
+ "# display(choice_df.head())"
+ ]
+ }
+ ],
+ "metadata": {
+ "interpreter": {
+ "hash": "ab0c6e94c9422d07d42069ec9e3bb23090f5e156fc0e23cc25ca45a62375bf53"
+ },
+ "kernelspec": {
+ "display_name": "emission",
+ "language": "python",
+ "name": "emission"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.16"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/replacement_mode_modeling/README.md b/replacement_mode_modeling/README.md
new file mode 100644
index 00000000..628fd439
--- /dev/null
+++ b/replacement_mode_modeling/README.md
@@ -0,0 +1,42 @@
+
+# Efforts towards predicting the replaced mode without user labels
+
+## Prerequisites:
+- These experiments were conducted on top of the `emission` anaconda environment. Please ensure that this environment is available to you before re-running the code.
+- In addition, some notebooks use `seaborn` for plotting and `pandarallel` for parallel pandas processing. The packages can be installed in the following manner:
+
+```
+(After activating emission conda env)
+pip3 install pandarallel==1.6.5
+pip3 install seaborn==0.12.2
+```
+
+- Ensure you have the following data sources loaded in your MongoDB Docker container:
+ - Stage_database (All CEO)
+ - Durham
+ - Masscec
+ - Ride2own
+ - UPRM NICR
+
+- Additionally, please also procure the CanBikeCO survey CSV file and place it in the `viz_scripts/` directory.
+
+- Once these data sources are procured and loaded in your Mongo container, you will need to add the inferred sections to the data. To do this, please run the [add_sections_and_summaries_to_trips.py](https://github.com/e-mission/e-mission-server/blob/master/bin/historical/migrations/add_sections_and_summaries_to_trips.py) script. **NOTE**: If you see a lot of errors in the log, try to re-run the script by modifying the following line from:
+
+```language=python
+# Before
+eps.dispatch(split_lists, skip_if_no_new_data=False, target_fn=add_sections_to_trips)
+
+# After
+eps.dispatch(split_lists, skip_if_no_new_data=False, target_fn=None)
+```
+
+This will trigger the intake pipeline for the current db and add the inferred section.
+
+- Note 2: The script above did not work for the All CEO data for me. Therefore, I obtained the section durations using the `get_section_durations` method I've written in the first notebook. Please note that running this script takes a long time and it is advised to cache the generated output.
+
+## Running the experiments
+The order in which the experiments are to be run are denoted by the preceding number. The following is a brief summary about each notebook:
+1. `01_extract_db_data.ipynb`: This notebook extracts the data, performs the necessary preprocessing, updates availability indicators, computes cost estimates, and stores the preprocessed data in `data/filtered_trips`.
+2. `02_run_trip_level_models.py`: This script reads all the preprocessed data, fits trip-level models with different stratitifications, generates the outputs, and stores them in `outputs/benchmark_results/`.
+3. `03_user_level_models.ipynb`: This notebook explores user fingerprints, similarity searching, and naive user-level models.
+4. `04_FeatureClustering.ipynb`: This notebook performs two functions: (a) Cluster users based on demographics/trip feature summaries and check for target distributions across clusters, and (b) Cluster users by grouping w.r.t. the target and checking for feature homogeneity within clusters
diff --git a/replacement_mode_modeling/data/README.md b/replacement_mode_modeling/data/README.md
new file mode 100644
index 00000000..6d2c55c4
--- /dev/null
+++ b/replacement_mode_modeling/data/README.md
@@ -0,0 +1 @@
+Temporary folder
\ No newline at end of file
diff --git a/replacement_mode_modeling/experimental_notebooks/LSTM.ipynb b/replacement_mode_modeling/experimental_notebooks/LSTM.ipynb
new file mode 100644
index 00000000..80260d7e
--- /dev/null
+++ b/replacement_mode_modeling/experimental_notebooks/LSTM.ipynb
@@ -0,0 +1,1398 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5f2cdb77",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import seaborn as sns\n",
+ "from abc import ABC, abstractmethod\n",
+ "from typing import List\n",
+ "import ast"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0ebc3879",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch\n",
+ "import random\n",
+ "import torch.nn as nn\n",
+ "import torch.nn.functional as F\n",
+ "import torch.optim as optim\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "from torch.utils.data import Dataset, DataLoader\n",
+ "from enum import Enum\n",
+ "import matplotlib.pyplot as plt\n",
+ "from torch.nn.utils.rnn import pad_sequence\n",
+ "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
+ "\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn.linear_model import LinearRegression\n",
+ "from sklearn.metrics import r2_score\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a2ace37f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Global experiment flags and variables.\n",
+ "SEED = 13210\n",
+ "\n",
+ "'''\n",
+ "'No Travel', 'Free Shuttle', 'Other', 'Gas Car, drove alone',\n",
+ " 'Regular Bike', 'Walk', 'Gas Car, with others', 'Bus', 'E-bike',\n",
+ " 'Scooter share', 'Taxi/Uber/Lyft', 'Train', 'Bikeshare',\n",
+ " 'Skate board', 'Not a Trip'\n",
+ "'''\n",
+ "\n",
+ "TARGET_MAPPING = {\n",
+ " 'No Travel': 'no_trip',\n",
+ " 'Free Shuttle': 'transit',\n",
+ " 'Other': 'unknown',\n",
+ " 'Gas Car, drove alone': 'car',\n",
+ " 'Regular Bike': 'p_micro',\n",
+ " 'Walk': 'walk',\n",
+ " 'Gas Car, with others': 's_micro',\n",
+ " 'Bus': 'transit',\n",
+ " 'E-bike': 'p_micro',\n",
+ " 'Scooter share': 's_micro',\n",
+ " 'Taxi/Uber/Lyft': 'ridehail',\n",
+ " 'Train': 'transit',\n",
+ " 'Bikeshare': 's_micro',\n",
+ " 'Skate board': 'p_micro',\n",
+ " 'Not a Trip': 'no_trip'\n",
+ "}\n",
+ "\n",
+ "\n",
+ "TARGETS = {\n",
+ " x: ix for (ix, x) in enumerate([\n",
+ " 'p_micro', 'no_trip', 's_car', 'transit', 'car', 's_micro', 'ridehail', 'walk', 'unknown'\n",
+ " ])\n",
+ "}\n",
+ "\n",
+ "av_modes = {\n",
+ " 'Skateboard': 'p_micro', \n",
+ " 'Walk/roll': 'walk', \n",
+ " 'Shared bicycle or scooter': 's_micro', \n",
+ " 'Taxi (regular taxi, Uber, Lyft, etc)': 'ridehail', \n",
+ " 'Rental car (including Zipcar/ Car2Go)': 'car',\n",
+ " 'Bicycle': 'p_micro', \n",
+ " 'Public transportation (bus, subway, light rail, etc.)': 'transit',\n",
+ " 'Get a ride from a friend or family member': 's_car',\n",
+ " 'None': 'no_trip', \n",
+ " 'Prefer not to say': 'unknown'\n",
+ "}\n",
+ "\n",
+ "# Set the Numpy seed too.\n",
+ "random.seed(SEED)\n",
+ "np.random.seed(SEED)\n",
+ "torch.manual_seed(SEED)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9addd580",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "TARGETS"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "481cc1bf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data = pd.read_csv('../data/final_modeling_data_02142024.csv')\n",
+ "weather_df = pd.read_csv('../data/denver_weather_data.csv')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8263d9ef",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data.Replaced_mode = data.Replaced_mode.replace(TARGET_MAPPING)\n",
+ "data.Replaced_mode = data.Replaced_mode.replace(TARGETS)\n",
+ "data.rename(columns={'Replaced_mode': 'target'}, inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8954515f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data[list(av_modes.values())] = 0"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bf9b787b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def encode_availability(x):\n",
+ " modes = [y.strip() for y in x.available_modes.split(';')]\n",
+ " mapped = set([av_modes[x] for x in modes])\n",
+ " \n",
+ " for mode in mapped:\n",
+ " x[mode] = 1\n",
+ " \n",
+ " return x\n",
+ "\n",
+ "\n",
+ "data = data.apply(lambda x: encode_availability(x), axis=1)\n",
+ "data.drop(columns=['available_modes'], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b010c95f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data['mark'] = 0\n",
+ "\n",
+ "data.section_distances = data.section_distances.apply(lambda x: ast.literal_eval(x))\n",
+ "data.section_modes = data.section_modes.apply(lambda x: ast.literal_eval(x))\n",
+ "data.section_durations = data.section_durations.apply(lambda x: ast.literal_eval(x))\n",
+ "\n",
+ "data.mark = data.apply(\n",
+ " lambda x: 1 if (len(x.section_distances) == len(x.section_modes) == len(x.section_durations))\n",
+ " and len(x.section_distances) > 0 and len(x.section_modes) > 0 and len(x.section_durations) > 0 else 0,\n",
+ " axis=1\n",
+ ")\n",
+ "\n",
+ "data.section_distances = data.section_distances.apply(lambda x: np.array(x).astype(np.float64))\n",
+ "data.section_modes = data.section_modes.apply(lambda x: np.array(x))\n",
+ "data.section_durations = data.section_durations.apply(lambda x: np.array(x).astype(np.float64))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0c79cdb8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data = data.loc[data.mark == 1, :].drop(columns=['mark'], inplace=False).reset_index(drop=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c420ee08",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SectionScaler:\n",
+ " def __init__(self):\n",
+ " self.dur = dict()\n",
+ " self.dist = dict()\n",
+ " \n",
+ " def compute_stats(self, df):\n",
+ " \n",
+ " for _, row in df[['section_modes', 'section_distances', 'section_durations']].iterrows():\n",
+ " for (mode, distance, duration) in zip(\n",
+ " row['section_modes'], row['section_distances'], row['section_durations']\n",
+ " ):\n",
+ " if mode not in self.dur.keys():\n",
+ " self.dur[mode] = [duration]\n",
+ " else:\n",
+ " self.dur[mode].append(duration)\n",
+ " \n",
+ " if mode not in self.dist.keys():\n",
+ " self.dist[mode] = [distance]\n",
+ " else:\n",
+ " self.dist[mode].append(distance)\n",
+ "\n",
+ " for mode in self.dur.keys():\n",
+ " self.dur[mode] = [np.nanmean(self.dur[mode]), np.std(self.dur[mode])]\n",
+ " \n",
+ " for mode in self.dist.keys():\n",
+ " self.dist[mode] = [np.nanmean(self.dist[mode]), np.std(self.dist[mode])]\n",
+ " \n",
+ " def apply(self, df):\n",
+ "\n",
+ " rows = list()\n",
+ " \n",
+ " for ix, x in df.iterrows():\n",
+ " row = x.to_dict()\n",
+ " modes = row['section_modes']\n",
+ " distances = row['section_distances']\n",
+ " durations = row['section_durations']\n",
+ " \n",
+ " norm_distances = [\n",
+ " (distances[i] - self.dist[mode][0])/self.dist[mode][1] for i, mode in enumerate(modes)\n",
+ " ]\n",
+ " \n",
+ " norm_durations = [\n",
+ " (durations[i] - self.dur[mode][0])/self.dur[mode][1] for i, mode in enumerate(modes)\n",
+ " ]\n",
+ "\n",
+ " if ix == 0:\n",
+ " print(norm_distances, norm_durations)\n",
+ " \n",
+ " row['section_distances'] = norm_distances\n",
+ " row['section_durations'] = norm_durations\n",
+ "\n",
+ " rows.append(row)\n",
+ "\n",
+ " return pd.DataFrame(data=rows)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "889bd770",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SPLIT_TYPE(Enum):\n",
+ " INTRA_USER = 0\n",
+ " INTER_USER = 1\n",
+ " TARGET = 2\n",
+ " MODE = 3\n",
+ " INTER_USER_STATIC = 4\n",
+ " \n",
+ "\n",
+ "class SPLIT(Enum):\n",
+ " TRAIN = 0\n",
+ " TEST = 1\n",
+ "\n",
+ "def get_splits(count_df: pd.DataFrame, n:int, test_size=0.2):\n",
+ " maxsize = int(n * test_size)\n",
+ "\n",
+ " max_threshold = int(maxsize * 1.05)\n",
+ " min_threshold = int(maxsize * 0.95)\n",
+ "\n",
+ " print(f\"{min_threshold}, {max_threshold}\")\n",
+ " \n",
+ " # Allow a 10% tolerance\n",
+ " def _dp(ix, curr_size, ids, cache):\n",
+ " \n",
+ " if ix >= count_df.shape[0]:\n",
+ " return []\n",
+ "\n",
+ " key = ix\n",
+ "\n",
+ " if key in cache:\n",
+ " return cache[key]\n",
+ "\n",
+ " if curr_size > max_threshold:\n",
+ " return []\n",
+ "\n",
+ " if min_threshold <= curr_size <= max_threshold:\n",
+ " return ids\n",
+ "\n",
+ " # two options - either pick the current id or skip it.\n",
+ " branch_a = _dp(ix, curr_size+count_df.loc[ix, 'count'], ids+[count_df.loc[ix, 'index']], cache)\n",
+ " branch_b = _dp(ix+1, curr_size, ids, cache)\n",
+ " \n",
+ " curr_max = []\n",
+ " if branch_a and len(branch_a) > 0:\n",
+ " curr_max = branch_a\n",
+ " \n",
+ " if branch_b and len(branch_b) > len(branch_a):\n",
+ " curr_max = branch_b\n",
+ " \n",
+ " cache[key] = curr_max\n",
+ " return cache[key]\n",
+ " \n",
+ " return _dp(0, 0, ids=list(), cache=dict())\n",
+ "\n",
+ "\n",
+ "def get_train_test_splits(data: pd.DataFrame, how=SPLIT_TYPE, test_ratio=0.2, shuffle=True):\n",
+ "\n",
+ " n_users = list(data.user_id.unique())\n",
+ " n = data.shape[0]\n",
+ " \n",
+ " if shuffle:\n",
+ " data = data.sample(data.shape[0], random_state=SEED).reset_index(drop=True, inplace=False)\n",
+ "\n",
+ " if how == SPLIT_TYPE.INTER_USER:\n",
+ " # Make the split, ensuring that a user in one fold is not leaked into the other fold.\n",
+ " # Basic idea: we want to start with the users with the highest instances and place \n",
+ " # alternating users in each set.\n",
+ " counts = data.user_id.value_counts().reset_index(drop=False, inplace=False, name='count')\n",
+ "\n",
+ " # Now, start with the user_id at the top, and keep adding to either split.\n",
+ " # This can be achieved using a simple DP program.\n",
+ " test_ids = get_splits(counts, data.shape[0])\n",
+ " test_data = data.loc[data.user_id.isin(test_ids), :]\n",
+ " train_index = data.index.difference(test_data.index)\n",
+ " train_data = data.loc[data.user_id.isin(train_index), :]\n",
+ " \n",
+ " return train_data, test_data\n",
+ " \n",
+ " elif how == SPLIT_TYPE.INTRA_USER:\n",
+ " \n",
+ " # There are certain users with only one observation. What do we do with those?\n",
+ " # As per the mobilitynet modeling pipeline, we randomly assign them to either the\n",
+ " # training or test set.\n",
+ " \n",
+ " value_counts = data.user_id.value_counts()\n",
+ " single_count_ids = value_counts[value_counts == 1].index\n",
+ " \n",
+ " data_filtered = data.loc[~data.user_id.isin(single_count_ids), :].reset_index(drop=True)\n",
+ " data_single_counts = data.loc[data.user_id.isin(single_count_ids), :].reset_index(drop=True)\n",
+ " \n",
+ " X_tr, X_te = train_test_split(\n",
+ " data_filtered, test_size=test_ratio, shuffle=shuffle, stratify=data_filtered.user_id,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " data_single_counts['assigned'] = np.random.choice(['train', 'test'], len(data_single_counts))\n",
+ " X_tr_merged = pd.concat(\n",
+ " [X_tr, data_single_counts.loc[data_single_counts.assigned == 'train', :].drop(\n",
+ " columns=['assigned'], inplace=False\n",
+ " )],\n",
+ " ignore_index=True, axis=0\n",
+ " )\n",
+ " \n",
+ " X_te_merged = pd.concat(\n",
+ " [X_te, data_single_counts.loc[data_single_counts.assigned == 'test', :].drop(\n",
+ " columns=['assigned'], inplace=False\n",
+ " )],\n",
+ " ignore_index=True, axis=0\n",
+ " )\n",
+ " \n",
+ " return X_tr_merged, X_te_merged\n",
+ " \n",
+ " elif how == SPLIT_TYPE.TARGET:\n",
+ " \n",
+ " X_tr, X_te = train_test_split(\n",
+ " data, test_size=test_ratio, shuffle=shuffle, stratify=data.target,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " return X_tr, X_te\n",
+ " \n",
+ " elif how == SPLIT_TYPE.MODE:\n",
+ " X_tr, X_te = train_test_split(\n",
+ " data, test_size=test_ratio, shuffle=shuffle, stratify=data.section_mode_argmax,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " return X_tr, X_te\n",
+ " \n",
+ " elif how == SPLIT_TYPE.INTER_USER_STATIC:\n",
+ " \n",
+ " train_ids = ['810be63d084746e3b7da9d943dd88e8c', 'bf774cbe6c3040b0a022278d36a23f19', '8a8332a53a1b4cdd9f3680434e91a6ef', \n",
+ " '5ad862e79a6341f69f28c0096fe884da', '7f89656bd4a94d12ad8e5ad9f0afecaf', 'fbaa338d7cd7457c8cad4d0e60a44d18', \n",
+ " '3b25446778824941a4c70ae5774f4c68', '28cb1dde85514bbabfd42145bdaf7e0a', '3aeb5494088542fdaf798532951aebb0', \n",
+ " '531732fee3c24366a286d76eb534aebc', '950f4287bab5444aa0527cc23fb082b2', '737ef8494f26407b8b2a6b1b1dc631a4', \n",
+ " 'e06cf95717f448ecb81c440b1b2fe1ab', '7347df5e0ac94a109790b31ba2e8a02a', 'bd9cffc8dbf1402da479f9f148ec9e60', \n",
+ " '2f3b66a5f98546d4b7691fba57fa640f', 'f289f7001bd94db0b33a7d2e1cd28b19', '19a043d1f2414dbcafcca44ea2bd1f19', \n",
+ " '68788082836e4762b26ad0877643fdcf', '4e8b1b7f026c4384827f157225da13fa', '703a9cee8315441faff7eb63f2bfa93f', \n",
+ " 'add706b73839413da13344c355dde0bb', '47b5d57bd4354276bb6d2dcd1438901d', 'e4cfb2a8f600426897569985e234636e', \n",
+ " '0154d71439284c34b865e5a417cd48af', '234f4f2366244fe682dccded2fa7cc4e', '0d0ae3a556414d138c52a6040a203d24', \n",
+ " '44c10f66dec244d6b8644231d4a8fecb', '30e9b141d7894fbfaacecd2fa18929f9', '0eb313ab00e6469da78cc2d2e94660fb', \n",
+ " 'fc51d1258e4649ecbfb0e6ecdaeca454', 'a1954793b1454b2f8cf95917d7547169', '6656c04c6cba4c189fed805eaa529741', \n",
+ " '6a0f3653b80a4c949e127d6504debb55', 'dfe5ca1bb0854b67a6ffccad9565d669', '8b1f3ba43de945bea79de6a81716ad04', \n",
+ " 'cde34edb8e3a4278a18e0adb062999e5', '6d96909e5ca442ccb5679d9cdf3c8f5b', 'a60a64d82d1c439a901b683b73a74d73', \n",
+ " '60e6a6f6ed2e4e838f2bbed6a427028d', '88041eddad7542ea8c92b30e5c64e198', '1635c003b1f94a399ebebe21640ffced', \n",
+ " '1581993b404a4b9c9ca6b0e0b8212316', 'b1aed24c863949bfbfa3a844ecf60593', '4b89612d7f1f4b368635c2bc48bd7993', \n",
+ " 'eb2e2a5211564a9290fcb06032f9b4af', '26767f9f3da54e93b692f8be6acdac43', '8a98e383a2d143e798fc23869694934a', \n",
+ " 'b346b83b9f7c4536b809d5f92074fdae', 'd929e7f8b7624d76bdb0ec9ada6cc650', '863e9c6c8ec048c4b7653f73d839c85b', \n",
+ " 'f50537eb104e4213908f1862c8160a3e', '4a9db5a9bac046a59403b44b883cc0ba', 'cded005d5fd14c64a5bba3f5c4fe8385', \n",
+ " 'c7ce889c796f4e2a8859fa2d7d5068fe', '405b221abe9e43bc86a57ca7fccf2227', '0b3e78fa91d84aa6a3203440143c8c16', \n",
+ " 'fbff5e08b7f24a94ab4b2d7371999ef7', 'e35e65107a34496db49fa5a0b41a1e9e', 'd5137ebd4f034dc193d216128bb7fc9a', \n",
+ " '3f7f2e536ba9481e92f8379b796ad1d0', 'dc75e0b776214e1b9888f6abd042fd95', 'b41dd7d7c6d94fe6afe2fd26fa4ac0bd', \n",
+ " 'eec6936e1ac347ef9365881845ec74df', '8c7d261fe8284a42a777ffa6f380ba3b', '4baf8c8af7b7445e9067854065e3e612', \n",
+ " 'c6e4db31c18b4355b02a7dd97deca70b', 'f0db3b1999c2410ba5933103eca9212f', '487e20ab774742378198f94f5b5b0b43', \n",
+ " 'dc1ed4d71e3645d0993885398d5628ca', '8c3c63abb3ec4fc3a61e7bf316ee4efd', '15eb78dd6e104966ba6112589c29dc41', \n",
+ " 'c23768ccb817416eaf08be487b2e3643', 'ecd2ae17d5184807abd87a287115c299', '71f21d53b655463784f3a3c63c56707b', \n",
+ " '2931e0a34319495bbb5898201a54feb5', '92bde0d0662f45ac864629f486cffe77', '42b3ee0bc02a481ab1a94644a8cd7a0d', \n",
+ " '15aa4ba144a34b8b8079ed7e049d84df', '509b909390934e988eb120b58ed9bd8c', '14103cda12c94642974129989d39e50d', \n",
+ " '8b0876430c2641bcaea954ea00520e64', 'baa4ff1573ae411183e10aeb17c71c53', '14fe8002bbdc4f97acbd1a00de241bf6', \n",
+ " '1b7d6dfea8464bcab9321018b10ec9c9', '487ad897ba93404a8cbe5de7d1922691', '5182d93d69754d7ba06200cd1ac5980a', \n",
+ " '91f3ca1c278247f79a806e49e9cc236f', 'e66e63b206784a559d977d4cb5f1ec34', '840297ae39484e26bfebe83ee30c5b3e', \n",
+ " 'c6807997194c4c528a8fa8c1f6ee1595', '802667b6371f45b29c7abb051244836a', 'b2bbe715b6a14fd19f751cae8adf6b4e', \n",
+ " 'feb1d940cd3647d1a101580c2a3b3f8c', '1b9883393ab344a69bc1a0fab192a94c', 'ac604b44fdca482fb753034cb55d1351', \n",
+ " 'f446bf3102ff4bd99ea1c98f7d2f7af0', 'c2c5d4b9a607487ea405a99c721079d4', '85ddd3c34c58407392953c47a32f5428', \n",
+ " 'd51de709f95045f8bacf473574b96ba5', '6373dfb8cb9b47e88e8f76adcfadde20', '313d003df34b4bd9823b3474fc93f9f9', \n",
+ " '53e78583db87421f8decb529ba859ca4', '8fdc9b926a674a9ea07d91df2c5e06f2', '90480ac60a3d475a88fbdab0a003dd5d', \n",
+ " '7559c3f880f341e898a402eba96a855d', '19a4c2cf718d40588eb96ac25a566353', 'f4427cccaa9442b48b42bedab5ab648e', \n",
+ " 'e192b8a00b6c422296851c93785deaf7', '355e25bdfc244c5e85d358e39432bd44', 'a0c3a7b410b24e18995f63369a31d123', \n",
+ " '03a395b4d8614757bb8432b4984559b0', 'a2d48b05d5454d428c0841432c7467b6', '3d981e617b304afab0f21ce8aa6c9786', \n",
+ " '2cd5668ac9054e2eb2c88bb4ed94bc6d', 'd7a732f4a8644bcbb8dedfc8be242fb2', '367eb90b929d4f6e9470d15c700d2e3f', \n",
+ " 'e049a7b2a6cb44259f907abbb44c5abc', 'a231added8674bef95092b32bc254ac8', 'e88a8f520dde445484c0a9395e1a0599',\n",
+ " 'cba570ae38f341faa6257342727377b7', '97953af1b97d4e268c52e1e54dcf421a', 'd200a61757d84b1dab8fbac35ff52c28', \n",
+ " 'fc68a5bb0a7b4b6386b3f08a69ead36f', '4a8210aec25e443391efb924cc0e5f23', '903742c353ce42c3ad9ab039fc418816', \n",
+ " '2114e2a75304475fad06ad201948fbad', 'ac917eae407c4deb96625dd0dc2f2ba9', '3dddfb70e7cd40f18a63478654182e9a', \n",
+ " 'd3735ba212dd4c768e1675dca7bdcb6f', '7abe572148864412a33979592fa985fb', 'd3dff742d07942ca805c2f72e49e12c5' \n",
+ " ]\n",
+ " \n",
+ " X_tr = data.loc[data.user_id.isin(train_ids), :]\n",
+ " X_te = data.loc[~data.user_id.isin(train_ids), :]\n",
+ " \n",
+ " return X_tr, X_te\n",
+ " \n",
+ " raise NotImplementedError(\"Unknown split type\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7b34ced0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def drop_columns(df: pd.DataFrame):\n",
+ " to_drop = [\n",
+ " 'raw_trip',\n",
+ " 'start_ts',\n",
+ " 'start_loc',\n",
+ " 'start_place',\n",
+ " 'end_place',\n",
+ " 'cleaned_trip',\n",
+ " 'inferred_labels',\n",
+ " 'inferred_trip',\n",
+ " 'expectation',\n",
+ " 'confidence_threshold',\n",
+ " 'expected_trip',\n",
+ " 'user_input',\n",
+ " 'start:year',\n",
+ " 'start:month',\n",
+ " 'start:day',\n",
+ " 'start:hour',\n",
+ " 'start_local_dt_minute',\n",
+ " 'start_local_dt_second',\n",
+ " 'start_local_dt_weekday',\n",
+ " 'start_local_dt_timezone',\n",
+ " 'end:year',\n",
+ " 'end:month',\n",
+ " 'end:day',\n",
+ " 'end:hour',\n",
+ " 'end_local_dt_minute',\n",
+ " 'end_local_dt_second',\n",
+ " 'end_local_dt_weekday',\n",
+ " 'end_local_dt_timezone',\n",
+ " '_id',\n",
+ " 'metadata_write_ts',\n",
+ " 'additions',\n",
+ " 'mode_confirm',\n",
+ " 'purpose_confirm',\n",
+ " 'distance_miles',\n",
+ " 'Mode_confirm',\n",
+ " 'Trip_purpose',\n",
+ " 'original_user_id',\n",
+ " 'program',\n",
+ " 'opcode',\n",
+ " 'Timestamp',\n",
+ " 'birth_year',\n",
+ " 'gender_Man',\n",
+ " 'gender_Man;Nonbinary/genderqueer/genderfluid',\n",
+ " 'gender_Nonbinary/genderqueer/genderfluid',\n",
+ " 'gender_Prefer not to say',\n",
+ " 'gender_Woman',\n",
+ " 'gender_Woman;Nonbinary/genderqueer/genderfluid',\n",
+ " 'has_multiple_jobs_No',\n",
+ " 'has_multiple_jobs_Prefer not to say',\n",
+ " 'has_multiple_jobs_Yes',\n",
+ " \"highest_education_Bachelor's degree\",\n",
+ " 'highest_education_Graduate degree or professional degree',\n",
+ " 'highest_education_High school graduate or GED',\n",
+ " 'highest_education_Less than a high school graduate',\n",
+ " 'highest_education_Prefer not to say',\n",
+ " 'highest_education_Some college or associates degree',\n",
+ " 'primary_job_type_Full-time',\n",
+ " 'primary_job_type_Part-time',\n",
+ " 'primary_job_type_Prefer not to say',\n",
+ " 'is_overnight_trip',\n",
+ " 'n_working_residents',\n",
+ " 'start_lat',\n",
+ " 'start_lng',\n",
+ " 'end_lat',\n",
+ " 'end_lng',\n",
+ " 'source', 'end_ts', 'end_fmt_time', 'end_loc',\n",
+ " ]\n",
+ "\n",
+ " # Drop section_mode_argmax and available_modes.\n",
+ " return df.drop(\n",
+ " columns=to_drop, \n",
+ " inplace=False\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "904fa4dc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "processed = drop_columns(data)\n",
+ "\n",
+ "train_df, test_df = get_train_test_splits(data=processed, how=SPLIT_TYPE.INTER_USER_STATIC, shuffle=True)\n",
+ "\n",
+ "scaler = SectionScaler()\n",
+ "scaler.compute_stats(train_df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "44354097",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(scaler.dist)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8c4fee4f-0da8-4391-9528-ef5fd7837365",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_df = scaler.apply(train_df)\n",
+ "test_df = scaler.apply(test_df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "71eed47f-3a58-4072-8dbe-f5287084f4c4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_df.shape, test_df.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "30d39919",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_df.section_distances.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9ff1c96e-7f18-44bd-8df2-f92239114e9e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SectionEmbedding(nn.Module):\n",
+ " def __init__(self, input_dim, emb_dim=32):\n",
+ " super(SectionEmbedding, self).__init__()\n",
+ " self.dpt = nn.Dropout(0.2)\n",
+ " self.encoder = nn.Linear(input_dim, emb_dim)\n",
+ " self.decoder = nn.Linear(emb_dim, input_dim)\n",
+ " self.act = nn.LeakyReLU()\n",
+ " \n",
+ " def forward(self, x):\n",
+ " '''\n",
+ " Input will be a one-hot encoded matrix, where nrows=number of modes, ncols=input_dim\n",
+ " dim = (B, N, D)\n",
+ " \n",
+ " '''"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "97a8d6b2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class ReplacedModeDataset(Dataset):\n",
+ " def __init__(self, df: pd.DataFrame):\n",
+ " self.data = df\n",
+ " \n",
+ " def __len__(self):\n",
+ " return len(self.data.ix.unique())\n",
+ " \n",
+ " def __getitem__(self, ix):\n",
+ " \n",
+ " # Could be between 1 - 15.\n",
+ " sequence = self.data.loc[self.data.ix == ix, :]\n",
+ " \n",
+ " # Static features that do not vary with time.\n",
+ " demographic_features = ['n_residence_members', \n",
+ " 'primary_job_commute_time', 'income_category',\n",
+ " 'n_residents_u18', 'n_residents_with_license', 'n_motor_vehicles', 'age', \n",
+ " 'p_micro', 'walk', 's_micro', 'ridehail', 'car', 'transit', 's_car', 'no_trip', 'unknown',\n",
+ " 'has_drivers_license_No', 'has_drivers_license_Prefer not to say', 'has_drivers_license_Yes', \n",
+ " 'primary_job_description_Clerical or administrative support', 'primary_job_description_Custodial', \n",
+ " 'primary_job_description_Education', 'primary_job_description_Food service', \n",
+ " 'primary_job_description_Manufacturing, construction, maintenance, or farming', \n",
+ " 'primary_job_description_Medical/healthcare', 'primary_job_description_Other', \n",
+ " 'primary_job_description_Professional, managerial, or technical', \n",
+ " 'primary_job_description_Sales or service', 'primary_job_commute_mode_Active transport', \n",
+ " 'primary_job_commute_mode_Car transport', 'primary_job_commute_mode_Hybrid', \n",
+ " 'primary_job_commute_mode_Public transport', 'primary_job_commute_mode_Unknown', \n",
+ " 'primary_job_commute_mode_WFH', 'duration', 'distance']\n",
+ " \n",
+ " seq_features = ['section_distances', 'section_durations', 'section_modes', 'mph']\n",
+ " \n",
+ " weather_features = ['temperature_2m (°F)', \n",
+ " 'relative_humidity_2m (%)', 'dew_point_2m (°F)', 'rain (inch)', 'snowfall (inch)', \n",
+ " 'wind_speed_10m (mp/h)', 'wind_gusts_10m (mp/h)']\n",
+ " \n",
+ " return (\n",
+ " sequence[seq_features], sequence[demographic_features], \n",
+ " sequence[weather_features], sequence['target']\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d3b36058",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dset = ReplacedModeDataset(train_df)\n",
+ "\n",
+ "print(dset.__getitem__(20))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "02b78758",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_dset = CustomDataset(train_df)\n",
+ "test_dset = CustomDataset(test_df)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "627b6fa4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def collate(batch):\n",
+ " X, y = zip(*batch)\n",
+ " \n",
+ " seq_modes = [x[0] for x in X]\n",
+ " seq_metrics = [x[1] for x in X]\n",
+ " features = [x[-1] for x in X]\n",
+ "\n",
+ " padded_seq = pad_sequence([s for s in seq_modes], batch_first=True)\n",
+ " padded_metrics = pad_sequence([m for m in seq_metrics], batch_first=True)\n",
+ " lengths = [len(seq) for seq in seq_modes]\n",
+ " stacked_features = torch.stack(features)\n",
+ "\n",
+ " return (padded_seq, padded_metrics, stacked_features), torch.stack(y), lengths"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5ca34681",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_loader = DataLoader(train_dset, batch_size=16, collate_fn=collate, shuffle=True, drop_last=False)\n",
+ "test_loader = DataLoader(test_dset, batch_size=16, collate_fn=collate, shuffle=True, drop_last=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31ca5ab1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "(modes, metrics, features), sY1, lX = next(iter(train_loader))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9eb5a93a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "metrics.size(), modes.size()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a0abf380",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Set to 0 for no dropout.\n",
+ "DROPOUT = 0."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "48871ea4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import math\n",
+ "\n",
+ "class GELU_new(nn.Module):\n",
+ " \"\"\"\n",
+ " Taken from OpenAI GPT-2 implementation.\n",
+ " \"\"\"\n",
+ " \n",
+ " def __init__(self):\n",
+ " super(GELU_new, self).__init__()\n",
+ " \n",
+ " def forward(self, x):\n",
+ " return 0.5 * x * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0))))\n",
+ "\n",
+ "\n",
+ "class DilatedBlock(nn.Module):\n",
+ " def __init__(self, n_c):\n",
+ " super(DilatedBlock, self).__init__()\n",
+ " \n",
+ " self.block = nn.Sequential(\n",
+ " nn.Linear(n_c, 4*n_c, bias=False),\n",
+ " GELU_new(),\n",
+ " nn.Linear(4*n_c, n_c, bias=False),\n",
+ " nn.Dropout(DROPOUT)\n",
+ " )\n",
+ " \n",
+ " def forward(self, x):\n",
+ " return self.block(x)\n",
+ "\n",
+ " \n",
+ "class SelfAttention(nn.Module):\n",
+ " def __init__(self, n_features, head_size):\n",
+ " super(SelfAttention, self).__init__()\n",
+ " # in: (B, F, 64)\n",
+ " self.k = nn.Linear(n_features, head_size, bias=False)\n",
+ " self.q = nn.Linear(n_features, head_size, bias=False)\n",
+ " self.v = nn.Linear(n_features, head_size, bias=False)\n",
+ " self.dpt = nn.Dropout(DROPOUT)\n",
+ " self.sqrt_d = torch.sqrt(torch.tensor(head_size))\n",
+ " \n",
+ " def forward(self, x):\n",
+ " k = self.k(x)\n",
+ " q = self.q(x)\n",
+ " v = self.v(x)\n",
+ " \n",
+ " # Q.K.t\n",
+ " dot = torch.bmm(q, k.permute(0, 2, 1))\n",
+ " \n",
+ " # normalize dot product.\n",
+ " dot /= self.sqrt_d\n",
+ " \n",
+ " # softmax over -1 dim.\n",
+ " softmax = self.dpt(torch.softmax(dot, dim=-1))\n",
+ " \n",
+ " # dot with values. (B, F, F) * (B, F, x) = (B, F, x)\n",
+ " return torch.bmm(softmax, v)\n",
+ " \n",
+ "\n",
+ "class MultiHeadAttention(nn.Module):\n",
+ " def __init__(self, n_heads, n_dim):\n",
+ " super(MultiHeadAttention, self).__init__()\n",
+ " \n",
+ " # 64 dims, 4 heads => 16 dims per head.\n",
+ " head_size = n_dim//n_heads\n",
+ " self.heads = nn.ModuleList([SelfAttention(n_dim, head_size) for _ in range(n_heads)])\n",
+ " self.proj = nn.Linear(n_dim, n_dim, bias=False)\n",
+ " \n",
+ " def forward(self, x):\n",
+ " # x is (B, seq, n_dim)\n",
+ " cat = torch.cat([head(x) for head in self.heads], dim=-1)\n",
+ " return self.proj(cat)\n",
+ "\n",
+ "\n",
+ "class Block(nn.Module):\n",
+ " def __init__(self, n_c):\n",
+ " super(Block, self).__init__()\n",
+ " \n",
+ " self.sa = MultiHeadAttention(n_heads=4, n_dim=n_c)\n",
+ " self.dilated = DilatedBlock(n_c)\n",
+ " self.ln1 = nn.LayerNorm(n_c)\n",
+ " self.ln2 = nn.LayerNorm(n_c)\n",
+ " \n",
+ " \n",
+ " def forward(self, x):\n",
+ " x = x + self.sa(self.ln1(x))\n",
+ " x = x + self.dilated(self.ln2(x))\n",
+ " return x\n",
+ " \n",
+ "\n",
+ "class LSTMLayer(nn.Module):\n",
+ " def __init__(\n",
+ " self, input_size: int, hidden_size: int, \n",
+ " output_size: int, n_lstm_layers: int = 1\n",
+ " ):\n",
+ " super(LSTMLayer, self).__init__()\n",
+ " \n",
+ " n_embed_mode = 16\n",
+ " \n",
+ " self.hidden_size = hidden_size\n",
+ " self.embedding = nn.Embedding(7, n_embed_mode, padding_idx=0)\n",
+ " self.dpt = nn.Dropout(DROPOUT)\n",
+ " \n",
+ " self.lstm = nn.LSTM(\n",
+ " input_size=input_size + n_embed_mode,\n",
+ " hidden_size=hidden_size,\n",
+ " bias=False,\n",
+ " bidirectional=True,\n",
+ " batch_first=True,\n",
+ " num_layers=n_lstm_layers\n",
+ " )\n",
+ " \n",
+ " def forward(self, modes, x, lengths):\n",
+ " mode_emb = self.embedding(modes)\n",
+ " x = torch.cat([x, mode_emb], dim=-1)\n",
+ " \n",
+ " packed = pack_padded_sequence(x, lengths, batch_first=True, enforce_sorted=False)\n",
+ " out, _ = self.lstm(packed)\n",
+ " unpacked, _ = pad_packed_sequence(out, batch_first=True)\n",
+ " \n",
+ " return self.dpt(unpacked)\n",
+ "\n",
+ "\n",
+ "class Model(nn.Module):\n",
+ " def __init__(\n",
+ " self, input_size: int, hidden_size: int, output_size: int, \n",
+ " n_features: int, n_lstm_layers: int = 1, **kwargs\n",
+ " ):\n",
+ " super(Model, self).__init__()\n",
+ " \n",
+ " block1_ip_dim = hidden_size*2\n",
+ " block2_ip_dim = (hidden_size*2) + n_features\n",
+ " \n",
+ " self.lstm = LSTMLayer(\n",
+ " input_size, hidden_size, \n",
+ " output_size, n_lstm_layers\n",
+ " )\n",
+ " \n",
+ " self.block_l1 = nn.ModuleList([Block(block1_ip_dim) for _ in range(kwargs['l1_blocks'])])\n",
+ " self.block_l2 = nn.ModuleList([Block(block2_ip_dim) for _ in range(kwargs['l2_blocks'])])\n",
+ " self.final_proj = nn.Linear(block2_ip_dim, output_size, bias=True)\n",
+ " \n",
+ " def forward(self, modes, x, features, lengths):\n",
+ " \n",
+ " b = x.size(0)\n",
+ " \n",
+ " # Out = (B, seq, hidden*2)\n",
+ " lstm_out = self.lstm(modes, x, lengths)\n",
+ " \n",
+ " # Pass the raw output through the blocks.\n",
+ " for module in self.block_l1:\n",
+ " lstm_out = module(lstm_out)\n",
+ " \n",
+ " features_rshp = features.unsqueeze(1).expand(b, lstm_out.size(1), -1)\n",
+ " \n",
+ " # Out = (B, seq, n+40)\n",
+ " cat = torch.cat([lstm_out, features_rshp], dim=-1)\n",
+ " \n",
+ " for module in self.block_l2:\n",
+ " cat = module(cat)\n",
+ " \n",
+ " # (8, 3, 104) -> (B, 104)\n",
+ " # flattened = cat.view(b, -1)\n",
+ " \n",
+ " # proj = self.runtime_ffw(flattened.size(-1), 64)(flattened)\n",
+ " proj = cat.mean(dim=1)\n",
+ " \n",
+ " return self.final_proj(proj)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "70b4d4ea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch.nn.init as init\n",
+ "\n",
+ "def init_weights(module):\n",
+ " if isinstance(module, nn.Embedding):\n",
+ " module.weight.data.normal_(mean=0.0, std=1.0)\n",
+ " if module.padding_idx is not None:\n",
+ " module.weight.data[module.padding_idx].zero_()\n",
+ " elif isinstance(module, nn.LayerNorm):\n",
+ " module.bias.data.zero_()\n",
+ " module.weight.data.fill_(1.0)\n",
+ " elif isinstance(module, nn.BatchNorm1d):\n",
+ " init.normal_(m.weight.data, mean=1, std=0.02)\n",
+ " init.constant_(m.bias.data, 0)\n",
+ " elif isinstance(module, nn.Linear):\n",
+ " init.xavier_normal_(module.weight.data)\n",
+ " if module.bias is not None:\n",
+ " init.normal_(module.bias.data)\n",
+ " elif isinstance(module, nn.LSTM):\n",
+ " for param in module.parameters():\n",
+ " if len(param.shape) >= 2:\n",
+ " init.orthogonal_(param.data)\n",
+ " else:\n",
+ " init.normal_(param.data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "282ecd1a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = Model(\n",
+ " n_lstm_layers=3,\n",
+ " input_size=3,\n",
+ " hidden_size=32, \n",
+ " output_size=num_classes,\n",
+ " n_features=40,\n",
+ " l1_blocks=4,\n",
+ " l2_blocks=4\n",
+ ")\n",
+ "\n",
+ "model = model.apply(init_weights)\n",
+ "\n",
+ "print(model)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "20fec22b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(sum(p.numel() for p in model.parameters()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1ca4b65a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "weights = train_df.shape[0]/(np.bincount(train_df.chosen.values) * len(np.unique(train_df.chosen)))\n",
+ "\n",
+ "print(weights)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e7a2017b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "INIT_LR = 1e-3\n",
+ "optimizer = optim.Adam(model.parameters(), lr=INIT_LR)\n",
+ "criterion = nn.CrossEntropyLoss(weight=torch.Tensor(weights))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a5bbda7c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class Trainer:\n",
+ " def __init__(self, model, tr_loader, te_loader):\n",
+ " pass\n",
+ " \n",
+ " def set_optim_params(self, **kwargs):\n",
+ " pass\n",
+ " \n",
+ " def set_criterion(self, **kwargs):\n",
+ " pass"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e53e4fd1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def train(epoch, model, loader, opt, criterion, val_ix):\n",
+ " \n",
+ " print(\"\\tBeginning training.\")\n",
+ " \n",
+ " n_batches = len(loader)\n",
+ " print_every = n_batches//5\n",
+ " \n",
+ " train_losses, val_losses = [], []\n",
+ " \n",
+ " for ix, (X, y, lengths) in enumerate(loader):\n",
+ " \n",
+ " # Unpack X.\n",
+ " modes, metrics, features = X\n",
+ " # Cast y to appropriate type.\n",
+ " y = y.float()\n",
+ " \n",
+ " if ix in val_ix:\n",
+ " model.eval()\n",
+ " with torch.no_grad():\n",
+ " y_pred = model(modes, metrics.float(), features.float(), lengths)\n",
+ " loss = criterion(y_pred.view(-1, num_classes), y.view(-1, num_classes))\n",
+ " val_losses.append(loss.item())\n",
+ " else:\n",
+ " model.train()\n",
+ " \n",
+ " opt.zero_grad()\n",
+ "\n",
+ " y_pred = model(modes, metrics.float(), features.float(), lengths)\n",
+ " loss = criterion(y_pred.view(-1, num_classes), y.view(-1, num_classes))\n",
+ " train_losses.append(loss.item())\n",
+ "\n",
+ " loss.backward()\n",
+ "\n",
+ " optimizer.step()\n",
+ " \n",
+ " if ix and ix % print_every == 0:\n",
+ " print(\n",
+ " f\"\\t-> Train loss: {np.nanmean(train_losses)}\\n\\t-> Val loss: {np.nanmean(val_losses)}\"\n",
+ " )\n",
+ " print('\\t'+20*'*')\n",
+ "\n",
+ " print(50*'-')\n",
+ " return train_losses, val_losses"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3a33fefa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def evaluate(model, loader, criterion):\n",
+ " \n",
+ " print(\"\\tBeginning evaluation.\")\n",
+ " \n",
+ " model.eval()\n",
+ " \n",
+ " print_every = len(loader)//5\n",
+ " \n",
+ " losses = []\n",
+ " \n",
+ " for ix, (X, y, lengths) in enumerate(loader):\n",
+ " \n",
+ " modes, metrics, features = X\n",
+ "\n",
+ " y_pred = model(modes, metrics.float(), features.float(), lengths)\n",
+ " y = y.float()\n",
+ " \n",
+ " loss = criterion(y_pred.view(-1, num_classes), y.view(-1, num_classes))\n",
+ "\n",
+ " losses.append(loss.item())\n",
+ " \n",
+ " if ix and ix % print_every == 0:\n",
+ " print(f\"\\t -> Average loss: {np.nanmean(losses)}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "650a5240",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.metrics import f1_score\n",
+ "\n",
+ "\n",
+ "def evaluate_f1(model, tr_loader, val_ix, te_loader=None):\n",
+ " \n",
+ " tr_preds, val_preds, te_preds = np.array([]), np.array([]), np.array([])\n",
+ " tr_gt, val_gt, te_gt = np.array([]), np.array([]), np.array([])\n",
+ " \n",
+ " model.eval()\n",
+ " print(\"\\tEvaluating F1...\")\n",
+ " \n",
+ " with torch.no_grad():\n",
+ " for ix, (X, y, lengths) in enumerate(tr_loader):\n",
+ " \n",
+ " modes, metrics, features = X\n",
+ "\n",
+ " y_pred = model(modes, metrics.float(), features.float(), lengths).view(-1, num_classes)\n",
+ " y = y.float().view(-1, num_classes)\n",
+ "\n",
+ " preds = torch.argmax(F.softmax(y_pred, dim=-1), dim=-1).numpy().ravel()\n",
+ " true = torch.argmax(y.long(), dim=-1).numpy().ravel()\n",
+ " \n",
+ " if ix in val_ix:\n",
+ " val_preds = np.append(val_preds, preds)\n",
+ " val_gt = np.append(val_gt, true)\n",
+ " else:\n",
+ " tr_preds = np.append(tr_preds, preds)\n",
+ " tr_gt = np.append(tr_gt, true)\n",
+ " \n",
+ " tr_f1 = f1_score(y_true=tr_gt, y_pred=tr_preds, average='weighted')\n",
+ " val_f1 = f1_score(y_true=val_gt, y_pred=val_preds, average='weighted')\n",
+ " print(f\"\\t -> Train F1: {tr_f1}, Val F1: {val_f1}\")\n",
+ " \n",
+ " if not te_loader:\n",
+ " return tr_f1, val_f1, None\n",
+ "\n",
+ " for ix, (X, y, lengths) in enumerate(te_loader):\n",
+ " \n",
+ " modes, metrics, features = X\n",
+ "\n",
+ " y_pred = model(modes, metrics.float(), features.float(), lengths).view(-1, num_classes)\n",
+ " y = y.float().view(-1, num_classes)\n",
+ " \n",
+ " preds = torch.argmax(F.softmax(y_pred, dim=-1), dim=-1).numpy().ravel()\n",
+ " true = torch.argmax(y.long(), dim=-1).numpy().ravel()\n",
+ "\n",
+ " te_preds = np.append(te_preds, preds)\n",
+ " te_gt = np.append(te_gt, true)\n",
+ " \n",
+ " te_f1 = f1_score(y_true=te_gt, y_pred=te_preds, average='weighted')\n",
+ " print(f\"\\t -> Test F1: {te_f1}\")\n",
+ " \n",
+ " return tr_f1, val_f1, te_f1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7191e78b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Other training hyperparameters.\n",
+ "num_epochs = 18\n",
+ "num_decays = 6\n",
+ "decay_at = num_epochs // num_decays\n",
+ "decay = 0.9\n",
+ "eval_every = 3\n",
+ "\n",
+ "# Static hold-out val set.\n",
+ "n_batches = len(train_loader)\n",
+ "val_split = 0.2\n",
+ "val_batches = np.random.choice(n_batches, size=(int(val_split * n_batches),), replace=False)\n",
+ "\n",
+ "# Just checking what LRs should be after decaying.\n",
+ "for power in range(num_decays):\n",
+ " print(f\"{decay_at * power} - {decay_at * (power + 1)} :: {INIT_LR * decay**power:.5f}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fc4b72de",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# We'd like to start at a loss of at most -ln(1/9) ~ 2.19\n",
+ "\n",
+ "# Wrapper to contain all losses.\n",
+ "tr_losses, val_losses = list(), list()\n",
+ "save_at_best_loss = True\n",
+ "best_loss = np.inf\n",
+ "model_name = \"../models/LSTM_{epoch}_{loss}.pt\"\n",
+ "patience, delta = 2, 0\n",
+ "\n",
+ "for epoch_ix in range(1, num_epochs+1):\n",
+ " print(f\"Epoch {epoch_ix}:\")\n",
+ " tr_loss, val_loss = train(epoch_ix, model, train_loader, optimizer, criterion, val_batches)\n",
+ " \n",
+ " tr_losses.extend(tr_loss)\n",
+ " val_losses.extend(val_loss)\n",
+ " \n",
+ " mean_val_loss = np.nanmean(val_loss)\n",
+ " \n",
+ " if epoch_ix and epoch_ix % eval_every == 0:\n",
+ " # evaluate(epoch_ix, model, test_loader, criterion)\n",
+ " tr_f1, val_f1, _ = evaluate_f1(model, train_loader, val_batches)\n",
+ " \n",
+ " if mean_val_loss < best_loss and save_at_best_loss:\n",
+ " best_loss = mean_val_loss\n",
+ " \n",
+ " # Reset delta.\n",
+ " delta = 0\n",
+ " \n",
+ " loss_str = str(best_loss).replace(\".\", \"_\")\n",
+ " torch.save(model.state_dict(), model_name.format(epoch=str(epoch_ix), loss=loss_str))\n",
+ " print(\"\\tSaved model checkpoint.\")\n",
+ " else:\n",
+ " # Increase delta by 1.\n",
+ " delta += 1\n",
+ " print(f\"\\tLoss did not decrease. Status is now {delta}/{patience}\")\n",
+ " \n",
+ " # Tolerate for `patience` epochs.\n",
+ " if delta == patience + 1:\n",
+ " # Stop training.\n",
+ " break\n",
+ "\n",
+ " if epoch_ix % decay_at == 0:\n",
+ " optimizer.param_groups[0]['lr'] *= decay\n",
+ " print(f\"\\tLearning rate is now: {optimizer.param_groups[0]['lr']:.5f}\")\n",
+ " \n",
+ " print(50*'-')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2bd1ffc7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Evaluate once on the test set.\n",
+ "evaluate(model, test_loader, criterion)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "396b615f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "final_tr_f1, final_val_f1, te_f1 = evaluate_f1(model, train_loader, val_batches, test_loader)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8bcc396c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# fig, ax = plt.subplots(figsize=(10, 6))\n",
+ "# ax.plot(tr_losses, 'r-')\n",
+ "# ax.plot(val_losses, 'b-')\n",
+ "# ax.set_title('Training and Validation losses')\n",
+ "# plt.legend(['Training loss', 'Validation loss'], loc='best')\n",
+ "# plt.tight_layout()\n",
+ "# plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a7d53498",
+ "metadata": {},
+ "source": [
+ "## Benchmarking\n",
+ "\n",
+ "\n",
+ "\n",
+ "epochs = 30\n",
+ "\n",
+ "```\n",
+ "LR scheme:\n",
+ "0 - 5 :: 0.00070\n",
+ "5 - 10 :: 0.00067\n",
+ "10 - 15 :: 0.00063\n",
+ "15 - 20 :: 0.00060\n",
+ "20 - 25 :: 0.00057\n",
+ "25 - 30 :: 0.00054\n",
+ "```\n",
+ "\n",
+ "```language=python\n",
+ "model = Model(\n",
+ " n_lstm_layers=1,\n",
+ " input_size=3,\n",
+ " hidden_size=16, \n",
+ " output_size=9,\n",
+ " n_features=40,\n",
+ " l1_blocks=6,\n",
+ " l2_blocks=6\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "\\# params: ~450k\n",
+ "\n",
+ "mode_embedding = 4\n",
+ "\n",
+ "Best stats:\n",
+ "\t -> Train F1: 0.7047532574096045\n",
+ "\t -> Test F1: 0.6560129685481192\n",
+ "\n",
+ "
\n",
+ "\n",
+ "epochs = 40\n",
+ "\n",
+ "Same LR scheme as above.\n",
+ "\n",
+ "```language=python\n",
+ "model = Model(\n",
+ " n_lstm_layers=3,\n",
+ " input_size=3,\n",
+ " hidden_size=32, \n",
+ " output_size=9,\n",
+ " n_features=40,\n",
+ " l1_blocks=4,\n",
+ " l2_blocks=4\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "\\# params: 770k\n",
+ "\n",
+ "mode_embedding = 4\n",
+ "\n",
+ "Best stats:\n",
+ "\t -> Train F1: 0.7365035440256072\n",
+ "\t -> Test F1: 0.6610215030981759\n",
+ " \n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "46a8dc7d",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "pytorch",
+ "language": "python",
+ "name": "pytorch"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.16"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/replacement_mode_modeling/experimental_notebooks/README.md b/replacement_mode_modeling/experimental_notebooks/README.md
new file mode 100644
index 00000000..13bb9e83
--- /dev/null
+++ b/replacement_mode_modeling/experimental_notebooks/README.md
@@ -0,0 +1,3 @@
+# All these scripts and notebooks are not verified to run,
+
+I am simply pushing these in this directory so that it might be of help for reference or to know what has already been tried. Please do not expect these notebooks to run seamlessly since a lot of them rely on intermediate pre-processed data.
\ No newline at end of file
diff --git a/replacement_mode_modeling/experimental_notebooks/baseline_modeling0.ipynb b/replacement_mode_modeling/experimental_notebooks/baseline_modeling0.ipynb
new file mode 100644
index 00000000..54472292
--- /dev/null
+++ b/replacement_mode_modeling/experimental_notebooks/baseline_modeling0.ipynb
@@ -0,0 +1,1011 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Notebook used for extensive experimentation on trip-level models with AllCEO data"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### All experiments are logged in Notion [here](https://www.notion.so/Replacement-mode-modeling-257c2f460377498d921e6b167f465945)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "from pathlib import Path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from enum import Enum\n",
+ "import random\n",
+ "\n",
+ "# Math and graphing.\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import seaborn as sns\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "# sklearn imports.\n",
+ "from sklearn.model_selection import train_test_split, StratifiedGroupKFold, GroupKFold\n",
+ "from sklearn.preprocessing import StandardScaler\n",
+ "from sklearn.linear_model import LinearRegression\n",
+ "from sklearn.metrics import f1_score, r2_score, ConfusionMatrixDisplay\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Global experiment flags and variables.\n",
+ "SEED = 19348\n",
+ "TARGETS = ['p_micro', 'no_trip', 's_car', 'transit', 'car', 's_micro', 'ridehail', 'walk', 'unknown']\n",
+ "\n",
+ "DROP_S_MICRO = True\n",
+ "\n",
+ "# Set the Numpy seed too.\n",
+ "random.seed(SEED)\n",
+ "np.random.seed(SEED)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SPLIT_TYPE(Enum):\n",
+ " INTRA_USER = 0\n",
+ " INTER_USER = 1\n",
+ " TARGET = 2\n",
+ " MODE = 3\n",
+ " INTER_USER_STATIC = 4\n",
+ " \n",
+ "\n",
+ "class SPLIT(Enum):\n",
+ " TRAIN = 0\n",
+ " TEST = 1\n",
+ "\n",
+ "\n",
+ "def get_train_test_splits(data: pd.DataFrame, how=SPLIT_TYPE, test_ratio=0.2, shuffle=True):\n",
+ " \n",
+ " if how == SPLIT_TYPE.INTER_USER:\n",
+ "\n",
+ " X = data.drop(columns=['target'])\n",
+ " y = data['target'].values\n",
+ " groups = data.user_id.values\n",
+ " \n",
+ " splitter = StratifiedGroupKFold(n_splits=5, shuffle=shuffle, random_state=SEED)\n",
+ " # splitter = GroupKFold(n_splits=5)\n",
+ " \n",
+ " for train_index, test_index in splitter.split(X, y, groups):\n",
+ " X_tr = data.iloc[train_index, :]\n",
+ " X_te = data.iloc[test_index, :]\n",
+ " \n",
+ " # Iterate only once and break.\n",
+ " break\n",
+ "\n",
+ " return X_tr, X_te\n",
+ " \n",
+ " elif how == SPLIT_TYPE.INTRA_USER:\n",
+ " \n",
+ " # There are certain users with only one observation. What do we do with those?\n",
+ " # As per the mobilitynet modeling pipeline, we randomly assign them to either the\n",
+ " # training or test set.\n",
+ " \n",
+ " value_counts = data.user_id.value_counts()\n",
+ " single_count_ids = value_counts[value_counts == 1].index\n",
+ " \n",
+ " data_filtered = data.loc[~data.user_id.isin(single_count_ids), :].reset_index(drop=True)\n",
+ " data_single_counts = data.loc[data.user_id.isin(single_count_ids), :].reset_index(drop=True)\n",
+ " \n",
+ " X_tr, X_te = train_test_split(\n",
+ " data_filtered, test_size=test_ratio, shuffle=shuffle, stratify=data_filtered.user_id,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " data_single_counts['assigned'] = np.random.choice(['train', 'test'], len(data_single_counts))\n",
+ " X_tr_merged = pd.concat(\n",
+ " [X_tr, data_single_counts.loc[data_single_counts.assigned == 'train', :].drop(\n",
+ " columns=['assigned'], inplace=False\n",
+ " )],\n",
+ " ignore_index=True, axis=0\n",
+ " )\n",
+ " \n",
+ " X_te_merged = pd.concat(\n",
+ " [X_te, data_single_counts.loc[data_single_counts.assigned == 'test', :].drop(\n",
+ " columns=['assigned'], inplace=False\n",
+ " )],\n",
+ " ignore_index=True, axis=0\n",
+ " )\n",
+ " \n",
+ " return X_tr_merged, X_te_merged\n",
+ " \n",
+ " elif how == SPLIT_TYPE.TARGET:\n",
+ " \n",
+ " X_tr, X_te = train_test_split(\n",
+ " data, test_size=test_ratio, shuffle=shuffle, stratify=data.target,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " return X_tr, X_te\n",
+ " \n",
+ " elif how == SPLIT_TYPE.MODE:\n",
+ " X_tr, X_te = train_test_split(\n",
+ " data, test_size=test_ratio, shuffle=shuffle, stratify=data.section_mode_argmax,\n",
+ " random_state=SEED\n",
+ " )\n",
+ " \n",
+ " return X_tr, X_te\n",
+ " \n",
+ " elif how == SPLIT_TYPE.INTER_USER_STATIC:\n",
+ " \n",
+ " train_ids = ['810be63d084746e3b7da9d943dd88e8c', 'bf774cbe6c3040b0a022278d36a23f19', '8a8332a53a1b4cdd9f3680434e91a6ef', \n",
+ " '5ad862e79a6341f69f28c0096fe884da', '7f89656bd4a94d12ad8e5ad9f0afecaf', 'fbaa338d7cd7457c8cad4d0e60a44d18', \n",
+ " '3b25446778824941a4c70ae5774f4c68', '28cb1dde85514bbabfd42145bdaf7e0a', '3aeb5494088542fdaf798532951aebb0', \n",
+ " '531732fee3c24366a286d76eb534aebc', '950f4287bab5444aa0527cc23fb082b2', '737ef8494f26407b8b2a6b1b1dc631a4', \n",
+ " 'e06cf95717f448ecb81c440b1b2fe1ab', '7347df5e0ac94a109790b31ba2e8a02a', 'bd9cffc8dbf1402da479f9f148ec9e60', \n",
+ " '2f3b66a5f98546d4b7691fba57fa640f', 'f289f7001bd94db0b33a7d2e1cd28b19', '19a043d1f2414dbcafcca44ea2bd1f19', \n",
+ " '68788082836e4762b26ad0877643fdcf', '4e8b1b7f026c4384827f157225da13fa', '703a9cee8315441faff7eb63f2bfa93f', \n",
+ " 'add706b73839413da13344c355dde0bb', '47b5d57bd4354276bb6d2dcd1438901d', 'e4cfb2a8f600426897569985e234636e', \n",
+ " '0154d71439284c34b865e5a417cd48af', '234f4f2366244fe682dccded2fa7cc4e', '0d0ae3a556414d138c52a6040a203d24', \n",
+ " '44c10f66dec244d6b8644231d4a8fecb', '30e9b141d7894fbfaacecd2fa18929f9', '0eb313ab00e6469da78cc2d2e94660fb', \n",
+ " 'fc51d1258e4649ecbfb0e6ecdaeca454', 'a1954793b1454b2f8cf95917d7547169', '6656c04c6cba4c189fed805eaa529741', \n",
+ " '6a0f3653b80a4c949e127d6504debb55', 'dfe5ca1bb0854b67a6ffccad9565d669', '8b1f3ba43de945bea79de6a81716ad04', \n",
+ " 'cde34edb8e3a4278a18e0adb062999e5', '6d96909e5ca442ccb5679d9cdf3c8f5b', 'a60a64d82d1c439a901b683b73a74d73', \n",
+ " '60e6a6f6ed2e4e838f2bbed6a427028d', '88041eddad7542ea8c92b30e5c64e198', '1635c003b1f94a399ebebe21640ffced', \n",
+ " '1581993b404a4b9c9ca6b0e0b8212316', 'b1aed24c863949bfbfa3a844ecf60593', '4b89612d7f1f4b368635c2bc48bd7993', \n",
+ " 'eb2e2a5211564a9290fcb06032f9b4af', '26767f9f3da54e93b692f8be6acdac43', '8a98e383a2d143e798fc23869694934a', \n",
+ " 'b346b83b9f7c4536b809d5f92074fdae', 'd929e7f8b7624d76bdb0ec9ada6cc650', '863e9c6c8ec048c4b7653f73d839c85b', \n",
+ " 'f50537eb104e4213908f1862c8160a3e', '4a9db5a9bac046a59403b44b883cc0ba', 'cded005d5fd14c64a5bba3f5c4fe8385', \n",
+ " 'c7ce889c796f4e2a8859fa2d7d5068fe', '405b221abe9e43bc86a57ca7fccf2227', '0b3e78fa91d84aa6a3203440143c8c16', \n",
+ " 'fbff5e08b7f24a94ab4b2d7371999ef7', 'e35e65107a34496db49fa5a0b41a1e9e', 'd5137ebd4f034dc193d216128bb7fc9a', \n",
+ " '3f7f2e536ba9481e92f8379b796ad1d0', 'dc75e0b776214e1b9888f6abd042fd95', 'b41dd7d7c6d94fe6afe2fd26fa4ac0bd', \n",
+ " 'eec6936e1ac347ef9365881845ec74df', '8c7d261fe8284a42a777ffa6f380ba3b', '4baf8c8af7b7445e9067854065e3e612', \n",
+ " 'c6e4db31c18b4355b02a7dd97deca70b', 'f0db3b1999c2410ba5933103eca9212f', '487e20ab774742378198f94f5b5b0b43', \n",
+ " 'dc1ed4d71e3645d0993885398d5628ca', '8c3c63abb3ec4fc3a61e7bf316ee4efd', '15eb78dd6e104966ba6112589c29dc41', \n",
+ " 'c23768ccb817416eaf08be487b2e3643', 'ecd2ae17d5184807abd87a287115c299', '71f21d53b655463784f3a3c63c56707b', \n",
+ " '2931e0a34319495bbb5898201a54feb5', '92bde0d0662f45ac864629f486cffe77', '42b3ee0bc02a481ab1a94644a8cd7a0d', \n",
+ " '15aa4ba144a34b8b8079ed7e049d84df', '509b909390934e988eb120b58ed9bd8c', '14103cda12c94642974129989d39e50d', \n",
+ " '8b0876430c2641bcaea954ea00520e64', 'baa4ff1573ae411183e10aeb17c71c53', '14fe8002bbdc4f97acbd1a00de241bf6', \n",
+ " '1b7d6dfea8464bcab9321018b10ec9c9', '487ad897ba93404a8cbe5de7d1922691', '5182d93d69754d7ba06200cd1ac5980a', \n",
+ " '91f3ca1c278247f79a806e49e9cc236f', 'e66e63b206784a559d977d4cb5f1ec34', '840297ae39484e26bfebe83ee30c5b3e', \n",
+ " 'c6807997194c4c528a8fa8c1f6ee1595', '802667b6371f45b29c7abb051244836a', 'b2bbe715b6a14fd19f751cae8adf6b4e', \n",
+ " 'feb1d940cd3647d1a101580c2a3b3f8c', '1b9883393ab344a69bc1a0fab192a94c', 'ac604b44fdca482fb753034cb55d1351', \n",
+ " 'f446bf3102ff4bd99ea1c98f7d2f7af0', 'c2c5d4b9a607487ea405a99c721079d4', '85ddd3c34c58407392953c47a32f5428', \n",
+ " 'd51de709f95045f8bacf473574b96ba5', '6373dfb8cb9b47e88e8f76adcfadde20', '313d003df34b4bd9823b3474fc93f9f9', \n",
+ " '53e78583db87421f8decb529ba859ca4', '8fdc9b926a674a9ea07d91df2c5e06f2', '90480ac60a3d475a88fbdab0a003dd5d', \n",
+ " '7559c3f880f341e898a402eba96a855d', '19a4c2cf718d40588eb96ac25a566353', 'f4427cccaa9442b48b42bedab5ab648e', \n",
+ " 'e192b8a00b6c422296851c93785deaf7', '355e25bdfc244c5e85d358e39432bd44', 'a0c3a7b410b24e18995f63369a31d123', \n",
+ " '03a395b4d8614757bb8432b4984559b0', 'a2d48b05d5454d428c0841432c7467b6', '3d981e617b304afab0f21ce8aa6c9786', \n",
+ " '2cd5668ac9054e2eb2c88bb4ed94bc6d', 'd7a732f4a8644bcbb8dedfc8be242fb2', '367eb90b929d4f6e9470d15c700d2e3f', \n",
+ " 'e049a7b2a6cb44259f907abbb44c5abc', 'a231added8674bef95092b32bc254ac8', 'e88a8f520dde445484c0a9395e1a0599',\n",
+ " 'cba570ae38f341faa6257342727377b7', '97953af1b97d4e268c52e1e54dcf421a', 'd200a61757d84b1dab8fbac35ff52c28', \n",
+ " 'fc68a5bb0a7b4b6386b3f08a69ead36f', '4a8210aec25e443391efb924cc0e5f23', '903742c353ce42c3ad9ab039fc418816', \n",
+ " '2114e2a75304475fad06ad201948fbad', 'ac917eae407c4deb96625dd0dc2f2ba9', '3dddfb70e7cd40f18a63478654182e9a', \n",
+ " 'd3735ba212dd4c768e1675dca7bdcb6f', '7abe572148864412a33979592fa985fb', 'd3dff742d07942ca805c2f72e49e12c5' \n",
+ " ]\n",
+ " \n",
+ " X_tr = data.loc[data.user_id.isin(train_ids), :]\n",
+ " X_te = data.loc[~data.user_id.isin(train_ids), :]\n",
+ " \n",
+ " return X_tr, X_te\n",
+ " \n",
+ " raise NotImplementedError(\"Unknown split type\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Modeling"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read the data.\n",
+ "data = pd.read_csv('../data/ReplacedMode_Fix_02142024.csv')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if DROP_S_MICRO:\n",
+ " data.drop(\n",
+ " index=data.loc[data.target == 6, :].index,\n",
+ " inplace=True\n",
+ " )\n",
+ " \n",
+ " # Shift all values after 6 by -1\n",
+ " data.loc[data.target > 5, 'target'] -= 1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data.drop_duplicates(inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def plot_hist(df, features=None):\n",
+ " if not features:\n",
+ " # All features.\n",
+ " features = df.columns.tolist()\n",
+ " \n",
+ " n_features = len(features)\n",
+ " \n",
+ " ncols = 6\n",
+ " nrows = n_features//ncols if n_features%ncols == 0 else (n_features//ncols) + 1\n",
+ " \n",
+ " fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(10, 10))\n",
+ " for ix, ax in enumerate(axes.flatten()):\n",
+ " \n",
+ " if ix > n_features:\n",
+ " break\n",
+ " \n",
+ " df[features[ix]].hist(ax=ax)\n",
+ " ax.set(title=features[ix])\n",
+ " \n",
+ " plt.tight_layout()\n",
+ " plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# First, we map the user IDs to ints.\n",
+ "\n",
+ "# USERS = list(data.user_id.unique())\n",
+ "\n",
+ "# USER_MAP = {\n",
+ "# u: i+1 for (i, u) in enumerate(USERS)\n",
+ "# }\n",
+ "\n",
+ "# data['user_id'] = data['user_id'].apply(lambda x: USER_MAP[x])\n",
+ "\n",
+ "# data.rename(\n",
+ "# columns={'start_local_dt_weekday': 'start:DOW', 'end_local_dt_weekday': 'end:DOW'},\n",
+ "# inplace=True\n",
+ "# )\n",
+ "\n",
+ "# Drop the samples with chosen == no trip or chosen == unknown\n",
+ "# data.drop(index=data.loc[data.chosen.isin([2, 9])].index, inplace=True)\n",
+ "\n",
+ "# data.n_working_residents = data.n_working_residents.apply(lambda x: 0 if x < 0 else x)\n",
+ "\n",
+ "# Fix some age preprocessing issues.\n",
+ "# data.age = data.age.apply(lambda x: x if x < 100 else 2024-x)\n",
+ "\n",
+ "# Collapse 'train' and 'bus' into 'transit'\n",
+ "# data.loc[\n",
+ "# data.section_mode_argmax.isin(['train', 'bus']), 'section_mode_argmax'\n",
+ "# ] = 'transit'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# display(data.section_mode_argmax.value_counts())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# transit = data.loc[data.section_mode_argmax == 'transit', :].copy()\n",
+ "# transit['section_duration_argmax'] /= 60.\n",
+ "\n",
+ "# transit['mph'] = transit['section_distance_argmax']/transit['section_duration_argmax']\n",
+ "\n",
+ "# display(transit[['section_duration_argmax', 'section_distance_argmax', 'mph']].describe())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# import plotly.express as px\n",
+ "\n",
+ "# sp = data.loc[data.section_mode_argmax.isin(['car', 'transit', 'walking']), :]\n",
+ "# fig = px.line(sp, y='section_distance_argmax', color='section_mode_argmax')\n",
+ "# fig.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Close the figure above.\n",
+ "# plt.close()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_duration_estimate(df: pd.DataFrame, dset: SPLIT, model_dict: dict):\n",
+ " \n",
+ " X_features = ['section_distance_argmax', 'age']\n",
+ " \n",
+ " if 'mph' in df.columns:\n",
+ " X_features += ['mph']\n",
+ " \n",
+ " if dset == SPLIT.TRAIN and model_dict is None:\n",
+ " model_dict = dict()\n",
+ " \n",
+ " if dset == SPLIT.TEST and model_dict is None:\n",
+ " raise AttributeError(\"Expected model dict for testing.\")\n",
+ " \n",
+ " if dset == SPLIT.TRAIN:\n",
+ " for section_mode in df.section_mode_argmax.unique():\n",
+ " section_data = df.loc[df.section_mode_argmax == section_mode, :]\n",
+ " if section_mode not in model_dict:\n",
+ " model_dict[section_mode] = dict()\n",
+ "\n",
+ " model = LinearRegression(fit_intercept=True)\n",
+ "\n",
+ " X = section_data[\n",
+ " X_features\n",
+ " ]\n",
+ " Y = section_data[['section_duration_argmax']]\n",
+ "\n",
+ " model.fit(X, Y.values.ravel())\n",
+ "\n",
+ " r2 = r2_score(y_pred=model.predict(X), y_true=Y.values.ravel())\n",
+ " print(f\"Train R2 for {section_mode}: {r2}\")\n",
+ "\n",
+ " model_dict[section_mode]['model'] = model\n",
+ " \n",
+ " elif dset == SPLIT.TEST:\n",
+ " for section_mode in df.section_mode_argmax.unique():\n",
+ " section_data = df.loc[df.section_mode_argmax == section_mode, :]\n",
+ " X = section_data[\n",
+ " X_features\n",
+ " ]\n",
+ " Y = section_data[['section_duration_argmax']]\n",
+ " \n",
+ " y_pred = model_dict[section_mode]['model'].predict(X)\n",
+ " r2 = r2_score(y_pred=y_pred, y_true=Y.values.ravel())\n",
+ " print(f\"Test R2 for {section_mode}: {r2}\")\n",
+ " \n",
+ " # Create the new columns for the duration.\n",
+ " new_columns = ['p_micro','no_trip','s_car','transit','car','s_micro','ridehail','walk','unknown']\n",
+ " df[new_columns] = 0\n",
+ " df['temp'] = 0\n",
+ " \n",
+ " for section in df.section_mode_argmax.unique():\n",
+ " X_section = df.loc[df.section_mode_argmax == section, X_features]\n",
+ " \n",
+ " # broadcast to all columns.\n",
+ " df.loc[df.section_mode_argmax == section, 'temp'] = model_dict[section]['model'].predict(X_section)\n",
+ " \n",
+ " for c in new_columns:\n",
+ " df[c] = df['av_' + c] * df['temp']\n",
+ " \n",
+ " df.drop(columns=['temp'], inplace=True)\n",
+ " \n",
+ " df.rename(columns=dict([(x, 'tt_'+x) for x in new_columns]), inplace=True)\n",
+ " \n",
+ " # return model_dict, result_df\n",
+ " return model_dict, df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Now, we split the data.\n",
+ "train_data, test_data = get_train_test_splits(data=data, how=SPLIT_TYPE.INTER_USER)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# If split is inter-user, we should verify test size.\n",
+ "\n",
+ "n_tr, n_te = len(train_data.user_id.unique()), len(test_data.user_id.unique())\n",
+ "n_ex_tr, n_ex_te = train_data.shape[0], test_data.shape[0]\n",
+ "\n",
+ "print(n_tr/(n_tr+n_te))\n",
+ "print(n_ex_tr/(n_ex_tr+n_ex_te))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(train_data.columns.tolist())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "params, train_data = get_duration_estimate(train_data, SPLIT.TRAIN, None)\n",
+ "print(10 * \"-\")\n",
+ "_, test_data = get_duration_estimate(test_data, SPLIT.TEST, params)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_data.shape, test_data.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Some helper functions that will help ease redundancy in the code.\n",
+ "\n",
+ "def drop_columns(df: pd.DataFrame):\n",
+ " to_drop = [\n",
+ " 'source', 'end_ts', 'end_fmt_time', 'end_loc', 'raw_trip', 'start_ts', \n",
+ " 'start_fmt_time', 'start_loc', 'duration', 'distance', 'start_place', \n",
+ " 'end_place', 'cleaned_trip', 'inferred_labels', 'inferred_trip', 'expectation',\n",
+ " 'confidence_threshold', 'expected_trip', 'user_input', 'start:year', 'start:month', \n",
+ " 'start:day', 'start_local_dt_minute', 'start_local_dt_second', \n",
+ " 'start_local_dt_weekday', 'start_local_dt_timezone', 'end:year', 'end:month', 'end:day', \n",
+ " 'end_local_dt_minute', 'end_local_dt_second', 'end_local_dt_weekday', \n",
+ " 'end_local_dt_timezone', '_id', 'user_id', 'metadata_write_ts', 'additions', \n",
+ " 'mode_confirm', 'purpose_confirm', 'Mode_confirm', 'Trip_purpose', \n",
+ " 'original_user_id', 'program', 'opcode', 'Timestamp', 'birth_year', \n",
+ " 'available_modes', 'section_coordinates_argmax', 'section_mode_argmax'\n",
+ " ]\n",
+ " \n",
+ " # Drop section_mode_argmax and available_modes.\n",
+ " return df.drop(\n",
+ " columns=to_drop, \n",
+ " inplace=False\n",
+ " )\n",
+ "\n",
+ "\n",
+ "def scale_values(df: pd.DataFrame, split: SPLIT, scalers=None):\n",
+ " # Scale costs using StandardScaler.\n",
+ " costs = df[[c for c in df.columns if 'cost_' in c]].copy()\n",
+ " times = df[[c for c in df.columns if 'tt_' in c or 'duration' in c]].copy()\n",
+ " distances = df[[c for c in df.columns if 'distance' in c]]\n",
+ " \n",
+ " print(\n",
+ " \"Cost columns to be scaled: \", costs.columns,\"\\nTime columns to be scaled: \", times.columns, \\\n",
+ " \"\\nDistance columns to be scaled: \", distances.columns\n",
+ " )\n",
+ " \n",
+ " if split == SPLIT.TRAIN and scalers is None:\n",
+ " cost_scaler = StandardScaler()\n",
+ " tt_scaler = StandardScaler()\n",
+ " dist_scaler = StandardScaler()\n",
+ " \n",
+ " cost_scaled = pd.DataFrame(\n",
+ " cost_scaler.fit_transform(costs), \n",
+ " columns=costs.columns, \n",
+ " index=costs.index\n",
+ " )\n",
+ " \n",
+ " tt_scaled = pd.DataFrame(\n",
+ " tt_scaler.fit_transform(times),\n",
+ " columns=times.columns,\n",
+ " index=times.index\n",
+ " )\n",
+ " \n",
+ " dist_scaled = pd.DataFrame(\n",
+ " dist_scaler.fit_transform(distances),\n",
+ " columns=distances.columns,\n",
+ " index=distances.index\n",
+ " )\n",
+ " \n",
+ " elif split == SPLIT.TEST and scalers is not None:\n",
+ " \n",
+ " cost_scaler, tt_scaler, dist_scaler = scalers\n",
+ " \n",
+ " cost_scaled = pd.DataFrame(\n",
+ " cost_scaler.transform(costs), \n",
+ " columns=costs.columns, \n",
+ " index=costs.index\n",
+ " )\n",
+ " \n",
+ " tt_scaled = pd.DataFrame(\n",
+ " tt_scaler.transform(times), \n",
+ " columns=times.columns, \n",
+ " index=times.index\n",
+ " )\n",
+ " \n",
+ " dist_scaled = pd.DataFrame(\n",
+ " dist_scaler.transform(distances),\n",
+ " columns=distances.columns,\n",
+ " index=distances.index\n",
+ " )\n",
+ " \n",
+ " else:\n",
+ " raise NotImplementedError(\"Unknown split\")\n",
+ " \n",
+ " # Drop the original columns.\n",
+ " df.drop(\n",
+ " columns=costs.columns.tolist() + times.columns.tolist() + distances.columns.tolist(), \n",
+ " inplace=True\n",
+ " )\n",
+ " \n",
+ " df = df.merge(right=cost_scaled, left_index=True, right_index=True)\n",
+ " df = df.merge(right=tt_scaled, left_index=True, right_index=True)\n",
+ " df = df.merge(right=dist_scaled, left_index=True, right_index=True)\n",
+ " \n",
+ " return df, (cost_scaler, tt_scaler, dist_scaler)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# First, drop columns.\n",
+ "\n",
+ "train_data = drop_columns(train_data)\n",
+ "\n",
+ "# Scale cost.\n",
+ "# train_data, scalers = scale_values(train_data, SPLIT.TRAIN, None)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_data = drop_columns(test_data)\n",
+ "\n",
+ "# Scale cost.\n",
+ "# test_data, _ = scale_values(test_data, SPLIT.TEST, scalers)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(train_data.columns)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "len(train_data.target.unique())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# train_data.to_csv('../data/train.csv', index=False)\n",
+ "# test_data.to_csv('../data/test.csv', index=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.metrics import classification_report\n",
+ "from sklearn.model_selection import GridSearchCV, StratifiedKFold\n",
+ "from pprint import pprint\n",
+ "from sklearn.inspection import permutation_importance\n",
+ "from time import perf_counter"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Random Forest classifier"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "CV = False\n",
+ "SAVE_MODEL = True"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sklearn.ensemble import RandomForestClassifier\n",
+ "\n",
+ "# exp question - compute sample weights using user_id.\n",
+ "\n",
+ "rf_train = train_data.drop(columns=['target', \n",
+ " 'start_lat', 'start_lng', 'end_lat', 'end_lng'\n",
+ " ])\n",
+ "rf_test = test_data.drop(columns=['target', \n",
+ " 'start_lat', 'start_lng', 'end_lat', 'end_lng'\n",
+ " ])\n",
+ "\n",
+ "if CV:\n",
+ "\n",
+ " model = RandomForestClassifier(random_state=SEED)\n",
+ "\n",
+ " # We want to build bootstrapped trees that would not always use all the features.\n",
+ "\n",
+ " param_set2 = {\n",
+ " 'n_estimators': [150, 200, 250, 300],\n",
+ " 'min_samples_split': [2, 3, 4],\n",
+ " 'min_samples_leaf': [1, 2, 3],\n",
+ " 'class_weight': ['balanced_subsample'],\n",
+ " 'max_features': [None, 'sqrt'],\n",
+ " 'bootstrap': [True]\n",
+ " }\n",
+ "\n",
+ " cv_set2 = StratifiedKFold(n_splits=3, shuffle=True, random_state=SEED)\n",
+ "\n",
+ " clf_set2 = GridSearchCV(model, param_set2, cv=cv_set2, n_jobs=-1, scoring='f1_weighted', verbose=1)\n",
+ "\n",
+ " start = perf_counter()\n",
+ "\n",
+ " clf_set2.fit(\n",
+ " rf_train,\n",
+ " train_data.target.values.ravel()\n",
+ " )\n",
+ "\n",
+ " time_req = (perf_counter() - start)/60.\n",
+ "\n",
+ " best_model = clf_set2.best_estimator_\n",
+ "else:\n",
+ " best_model = RandomForestClassifier(\n",
+ " n_estimators=150,\n",
+ " max_depth=None,\n",
+ " min_samples_leaf=2,\n",
+ " bootstrap=True,\n",
+ " class_weight='balanced_subsample',\n",
+ " random_state=SEED,\n",
+ " n_jobs=-1\n",
+ " ).fit(rf_train, train_data.target.values.ravel())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "best_model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tr_f1_set2 = f1_score(\n",
+ " y_true=train_data.target.values,\n",
+ " y_pred=best_model.predict(rf_train),\n",
+ " average='weighted'\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "te_f1_set2 = f1_score(\n",
+ " y_true=test_data.target.values,\n",
+ " y_pred=best_model.predict(rf_test),\n",
+ " average='weighted'\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Without location:\n",
+ "#. intra-user split:\n",
+ "# [BOOTSTRAPPED] | Train F1: 0.9983454261487021, Test F1: 0.7192048995905516\n",
+ "# if stratified by section_mode_argmax:\n",
+ "# [BOOTSTRAPPED] | Train F1: 0.9987250576328509, Test F1: 0.7242573620109232\n",
+ "\n",
+ "# With location:\n",
+ "# [BOOTSTRAPPED] | Train F1: 0.9992402006853468, Test F1: 0.7654135199070202\n",
+ "\n",
+ "print(f\"[BOOTSTRAPPED] | Train F1: {tr_f1_set2}, Test F1: {te_f1_set2}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if SAVE_MODEL:\n",
+ "\n",
+ " import pickle\n",
+ "\n",
+ " with open('../models/tuned_rf_model.pkl', 'wb') as f:\n",
+ " f.write(pickle.dumps(best_model))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Feature importances - gini entropy\n",
+ "\n",
+ "pprint(\n",
+ " sorted(\n",
+ " zip(\n",
+ " best_model.feature_names_in_, \n",
+ " best_model.feature_importances_\n",
+ " ), \n",
+ " key=lambda x: x[-1], reverse=True\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# importance = permutation_importance(\n",
+ "# best_model,\n",
+ "# rf_test,\n",
+ "# test_data.target.values,\n",
+ "# n_repeats=5,\n",
+ "# random_state=SEED,\n",
+ "# n_jobs=-1,\n",
+ "# scoring='f1_weighted'\n",
+ "# )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# pd.DataFrame(\n",
+ "# {\n",
+ "# 'feature names': test_data.columns.delete(\n",
+ "# test_data.columns.isin(['target'])\n",
+ "# ),\n",
+ "# 'imp_mean': importance.importances_mean, \n",
+ "# 'imp_std': importance.importances_std\n",
+ "# }\n",
+ "# ).sort_values(by=['imp_mean'], axis='rows', ascending=False).head(20)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# fig, ax = plt.subplots(nrows=1, ncols=2)\n",
+ "y_pred = best_model.predict(rf_test)\n",
+ "pred_df = pd.DataFrame(\n",
+ " {\n",
+ " 'y_pred': y_pred.ravel(),\n",
+ " 'y_true': test_data.target.values.ravel()\n",
+ " }\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, ax = plt.subplots(figsize=(7, 7))\n",
+ "cm = ConfusionMatrixDisplay.from_estimator(\n",
+ " best_model,\n",
+ " X=rf_test,\n",
+ " y=test_data[['target']],\n",
+ " ax=ax\n",
+ ")\n",
+ "# ax.set_xticklabels(TARGETS, rotation=45)\n",
+ "# ax.set_yticklabels(TARGETS)\n",
+ "fig.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# print(classification_report(y_true=pred_df.y_true, y_pred=pred_df.y_pred, target_names=TARGETS))\n",
+ "print(classification_report(y_true=pred_df.y_true, y_pred=pred_df.y_pred))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## XGBoost"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# from sklearn.utils.class_weight import compute_sample_weight\n",
+ "\n",
+ "# sample_weights = compute_sample_weight(class_weight='balanced', y=train_data.user_id.values.ravel())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# from xgboost import XGBClassifier\n",
+ "\n",
+ "# y_train = train_data.target.values.ravel() - 1\n",
+ "# y_test = test_data.target.values.ravel() - 1\n",
+ "\n",
+ "# # weights = compute_class_weight(class_weight='balanced', classes=np.unique(y_pred), y_pred)\n",
+ "\n",
+ "# xgm = XGBClassifier(\n",
+ "# n_estimators=300,\n",
+ "# max_depth=None,\n",
+ "# tree_method='hist',\n",
+ "# objective='multi:softmax',\n",
+ "# num_class=9\n",
+ "# ).fit(rf_train, y_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# preds = xgm.predict(rf_test)\n",
+ "\n",
+ "# print(classification_report(y_true=y_test, y_pred=preds, target_names=TARGETS))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# import pickle\n",
+ "\n",
+ "# # RF_RM.pkl = 0.8625 on test.\n",
+ "# # RF_RM_1.pkl = 0.77 on test.\n",
+ "# with open('../models/RF_RM_1.pkl', 'wb') as f:\n",
+ "# f.write(pickle.dumps(model))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## TODO:\n",
+ "\n",
+ "\n",
+ "- Explain why location might not be a good feature to add (plot start and end on map and explain how model might just overfit to the raw coordinates)\n",
+ "- Merge `unknown` and `no_trip` into one category and validate against models trained on (a) separate labels (b) dropped labels\n",
+ "- Explore more of the abnormal `walking` trips"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "interpreter": {
+ "hash": "ab0c6e94c9422d07d42069ec9e3bb23090f5e156fc0e23cc25ca45a62375bf53"
+ },
+ "kernelspec": {
+ "display_name": "emission",
+ "language": "python",
+ "name": "emission"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.16"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/replacement_mode_modeling/experimental_notebooks/biogeme_modeling train_test_w_splits.ipynb b/replacement_mode_modeling/experimental_notebooks/biogeme_modeling train_test_w_splits.ipynb
new file mode 100644
index 00000000..5cc4f68a
--- /dev/null
+++ b/replacement_mode_modeling/experimental_notebooks/biogeme_modeling train_test_w_splits.ipynb
@@ -0,0 +1,1107 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Biogeme modeling for inter-user modeling. Contains outputs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "from enum import Enum\n",
+ "from sklearn.model_selection import train_test_split"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class SPLIT_TYPE(Enum):\n",
+ " INTRA_USER = 0\n",
+ " INTER_USER = 1\n",
+ " \n",
+ "\n",
+ "class SPLIT(Enum):\n",
+ " TRAIN = 0\n",
+ " TEST = 1\n",
+ "\n",
+ "\n",
+ "def get_splits(count_df: pd.DataFrame, n:int, test_size=0.2):\n",
+ " maxsize = int(n * test_size)\n",
+ "\n",
+ " max_threshold = int(maxsize * 1.05)\n",
+ " min_threshold = int(maxsize * 0.95)\n",
+ "\n",
+ " print(f\"{min_threshold=}, {max_threshold=}\")\n",
+ " \n",
+ " # Allow a 10% tolerance\n",
+ " def _dp(ix, curr_size, ids, cache):\n",
+ " \n",
+ " if ix >= count_df.shape[0]:\n",
+ " return []\n",
+ "\n",
+ " key = ix\n",
+ "\n",
+ " if key in cache:\n",
+ " return cache[key]\n",
+ "\n",
+ " if curr_size > max_threshold:\n",
+ " return []\n",
+ "\n",
+ " if min_threshold <= curr_size <= max_threshold:\n",
+ " return ids\n",
+ "\n",
+ " # two options - either pick the current id or skip it.\n",
+ " branch_a = _dp(ix, curr_size+count_df.loc[ix, 'count'], ids+[count_df.loc[ix, 'index']], cache)\n",
+ " branch_b = _dp(ix+1, curr_size, ids, cache)\n",
+ " \n",
+ " curr_max = []\n",
+ " if branch_a and len(branch_a) > 0:\n",
+ " curr_max = branch_a\n",
+ " \n",
+ " if branch_b and len(branch_b) > len(branch_a):\n",
+ " curr_max = branch_b\n",
+ " \n",
+ " cache[key] = curr_max\n",
+ " return cache[key]\n",
+ " \n",
+ " return _dp(0, 0, ids=list(), cache=dict())\n",
+ "\n",
+ "\n",
+ "def get_train_test_splits(data: pd.DataFrame, how=SPLIT_TYPE, test_ratio=0.2, shuffle=True):\n",
+ "\n",
+ " n_users = list(data.user_id.unique())\n",
+ " n = data.shape[0]\n",
+ " \n",
+ " if shuffle:\n",
+ " data = data.sample(data.shape[0]).reset_index(drop=True, inplace=False)\n",
+ "\n",
+ " if how == SPLIT_TYPE.INTER_USER:\n",
+ " # Make the split, ensuring that a user in one fold is not leaked into the other fold.\n",
+ " # Basic idea: we want to start with the users with the highest instances and place alternating users in each set.\n",
+ " counts = data.user_id.value_counts().reset_index(drop=False, inplace=False, name='count')\n",
+ "\n",
+ " # Now, start with the user_id at the top, and keep adding to either split.\n",
+ " # This can be achieved using a simple DP program.\n",
+ " test_ids = get_splits(counts, data.shape[0])\n",
+ " test_data = data.loc[data.user_id.isin(test_ids), :]\n",
+ " train_index = data.index.difference(test_data.index)\n",
+ " train_data = data.loc[data.user_id.isin(train_index), :]\n",
+ " \n",
+ " return train_data, test_data\n",
+ " \n",
+ " elif how == SPLIT_TYPE.INTRA_USER:\n",
+ " \n",
+ " # There are certain users with only one observation. What do we do with those?\n",
+ " # As per the mobilitynet modeling pipeline, we randomly assign them to either the\n",
+ " # training or test set.\n",
+ " \n",
+ " value_counts = data.user_id.value_counts()\n",
+ " single_count_ids = value_counts[value_counts == 1].index\n",
+ " \n",
+ " data_filtered = data.loc[~data.user_id.isin(single_count_ids), :].reset_index(drop=True)\n",
+ " data_single_counts = data.loc[data.user_id.isin(single_count_ids), :].reset_index(drop=True)\n",
+ " \n",
+ " X_tr, X_te = train_test_split(\n",
+ " data_filtered, test_size=test_ratio, shuffle=shuffle, stratify=data_filtered.user_id\n",
+ " )\n",
+ " \n",
+ " data_single_counts['assigned'] = np.random.choice(['train', 'test'], len(data_single_counts))\n",
+ " X_tr_merged = pd.concat(\n",
+ " [X_tr, data_single_counts.loc[data_single_counts.assigned == 'train', :].drop(\n",
+ " columns=['assigned'], inplace=False\n",
+ " )],\n",
+ " ignore_index=True, axis=0\n",
+ " )\n",
+ " \n",
+ " X_te_merged = pd.concat(\n",
+ " [X_te, data_single_counts.loc[data_single_counts.assigned == 'test', :].drop(\n",
+ " columns=['assigned'], inplace=False\n",
+ " )],\n",
+ " ignore_index=True, axis=0\n",
+ " )\n",
+ " \n",
+ " return X_tr_merged, X_te_merged\n",
+ " \n",
+ " raise NotImplementedError(\"Unknown split type\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Modeling"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import biogeme.biogeme as bio\n",
+ "import biogeme.database as db\n",
+ "from biogeme import models\n",
+ "from biogeme.expressions import Beta, DefineVariable\n",
+ "from biogeme.expressions import Variable\n",
+ "import numpy as np\n",
+ "\n",
+ "from sklearn.preprocessing import MinMaxScaler"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "n_rows: 164281\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Read the data.\n",
+ "data = pd.read_csv('../data/preprocessed_data_split_chosen.csv')\n",
+ "\n",
+ "print(\"n_rows: \", data.shape[0])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# First, we map the user IDs to ints.\n",
+ "\n",
+ "USER_MAP = {\n",
+ " u: i+1 for (i, u) in enumerate(data.user_id.unique())\n",
+ "}\n",
+ "\n",
+ "data['user_id'] = data['user_id'].apply(lambda x: USER_MAP[x])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Now, we split the data (either inter-user or intra-user split)\n",
+ "\n",
+ "# 0.98???\n",
+ "# train_data, test_data = get_train_test_splits(data=data, how=SPLIT_TYPE.INTER_USER, shuffle=True)\n",
+ "\n",
+ "# 0.975???\n",
+ "train_data, test_data = get_train_test_splits(data=data, how=SPLIT_TYPE.INTRA_USER, shuffle=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Some helper functions that will help ease redundancy in the code.\n",
+ "\n",
+ "def drop_columns(df: pd.DataFrame):\n",
+ " # Drop section_mode_argmax and available_modes.\n",
+ " return df.drop(columns=[\n",
+ " 'section_mode_argmax', 'available_modes', 'section_duration_argmax', 'section_distance_argmax'\n",
+ " ], inplace=False)\n",
+ "\n",
+ "\n",
+ "def scale_time(df: pd.DataFrame):\n",
+ " # Convert from min -> hrs\n",
+ " df[[c for c in df.columns if 'tt_' in c]] /= 60.\n",
+ " return df\n",
+ "\n",
+ "\n",
+ "def scale_cost(df: pd.DataFrame, split: SPLIT, scaler=None):\n",
+ " # Scale costs using MinMaxScaler.\n",
+ " costs = df[[c for c in df.columns if 'cost_' in c]].copy()\n",
+ " \n",
+ " if split == SPLIT.TRAIN and scaler is None:\n",
+ " scaler = MinMaxScaler()\n",
+ " cost_scaled = pd.DataFrame(\n",
+ " scaler.fit_transform(costs), \n",
+ " columns=['scaled_' + c for c in costs.columns], \n",
+ " index=costs.index\n",
+ " )\n",
+ " \n",
+ " elif split == SPLIT.TEST and scaler is not None:\n",
+ " cost_scaled = pd.DataFrame(\n",
+ " scaler.transform(costs), \n",
+ " columns=['scaled_' + c for c in costs.columns], \n",
+ " index=costs.index\n",
+ " )\n",
+ " \n",
+ " else:\n",
+ " raise NotImplementedError(\"Unknown split\")\n",
+ " \n",
+ " df = df.merge(right=cost_scaled, left_index=True, right_index=True)\n",
+ " \n",
+ " return df, scaler\n",
+ "\n",
+ "\n",
+ "def get_database(df: pd.DataFrame, split: SPLIT):\n",
+ " return db.Database(split.name + '_db', df)\n",
+ "\n",
+ "\n",
+ "def get_variables():\n",
+ " USER_ID = Variable('user_id')\n",
+ "\n",
+ " # Availability.\n",
+ " AV_P_MICRO = Variable('av_p_micro')\n",
+ " AV_NO_TRIP = Variable('av_no_trip')\n",
+ " AV_S_CAR = Variable('av_s_car')\n",
+ " AV_TRANSIT = Variable('av_transit')\n",
+ " AV_CAR = Variable('av_car')\n",
+ " AV_S_MICRO = Variable('av_s_micro')\n",
+ " AV_RIDEHAIL = Variable('av_ridehail')\n",
+ " AV_WALK = Variable('av_walk')\n",
+ " AV_UNKNOWN = Variable('av_unknown')\n",
+ "\n",
+ " # Time.\n",
+ " TT_P_MICRO = Variable('tt_p_micro')\n",
+ " TT_NO_TRIP = Variable('tt_no_trip')\n",
+ " TT_S_CAR = Variable('tt_s_car')\n",
+ " TT_TRANSIT = Variable('tt_transit')\n",
+ " TT_CAR = Variable('tt_car')\n",
+ " TT_S_MICRO = Variable('tt_s_micro')\n",
+ " TT_RIDEHAIL = Variable('tt_ridehail')\n",
+ " TT_WALK = Variable('tt_walk')\n",
+ " TT_UNKNOWN = Variable('tt_unknown')\n",
+ "\n",
+ " # Cost.\n",
+ " CO_P_MICRO = Variable('scaled_cost_p_micro')\n",
+ " CO_NO_TRIP = Variable('scaled_cost_no_trip')\n",
+ " CO_S_CAR = Variable('scaled_cost_s_car')\n",
+ " CO_TRANSIT = Variable('scaled_cost_transit')\n",
+ " CO_CAR = Variable('scaled_cost_car')\n",
+ " CO_S_MICRO = Variable('scaled_cost_s_micro')\n",
+ " CO_RIDEHAIL = Variable('scaled_cost_ridehail')\n",
+ " CO_WALK = Variable('scaled_cost_walk')\n",
+ " CO_UNKNOWN = Variable('scaled_cost_unknown')\n",
+ "\n",
+ " # Choice.\n",
+ " CHOICE = Variable('chosen')\n",
+ " \n",
+ " # return the filtered locals() dictionary.\n",
+ " return {k:v for k,v in locals().items() if not k.startswith('_')}\n",
+ "\n",
+ "\n",
+ "def exclude_from_db(v_dict: dict, db: db.Database):\n",
+ " EXCLUDE = (v_dict['CHOICE'] == 2) + (v_dict['CHOICE'] == 9) > 0\n",
+ " db.remove(EXCLUDE)\n",
+ "\n",
+ "\n",
+ "def get_params():\n",
+ " B_TIME = Beta('B_TIME', 0, None, 0, 0)\n",
+ " B_COST = Beta('B_COST', 0, None, None, 0)\n",
+ "\n",
+ " # Alternative-Specific Constants.\n",
+ " ASC_P_MICRO = Beta('ASC_P_MICRO', 0, None, None, 0)\n",
+ " ASC_NO_TRIP = Beta('ASC_NO_TRIP', 0, None, None, 0)\n",
+ " ASC_S_CAR = Beta('ASC_S_CAR', 0, None, None, 0)\n",
+ " ASC_TRANSIT = Beta('ASC_TRANSIT', 0, None, None, 0)\n",
+ " ASC_CAR = Beta('ASC_CAR', 0, None, None, 0)\n",
+ " ASC_S_MICRO = Beta('ASC_S_MICRO', 0, None, None, 0)\n",
+ " ASC_RIDEHAIL = Beta('ASC_RIDEHAIL', 0, None, None, 0)\n",
+ " ASC_WALK = Beta('ASC_WALK', 0, None, None, 0)\n",
+ " ASC_UNKNOWN = Beta('ASC_UNKNOWN', 0, None, None, 0)\n",
+ " \n",
+ " # Return filtered locals dict.\n",
+ " return {k:v for k,v in locals().items() if not k.startswith('_')}\n",
+ "\n",
+ "\n",
+ "def get_utility_functions(v: dict):\n",
+ " V_P_MICRO = (\n",
+ " v['ASC_P_MICRO'] +\n",
+ " v['B_TIME'] * v['TT_P_MICRO']\n",
+ " + v['B_COST'] * v['CO_P_MICRO']\n",
+ " )\n",
+ "\n",
+ " V_NO_TRIP = (\n",
+ " v['ASC_NO_TRIP'] +\n",
+ " v['B_TIME'] * v['TT_NO_TRIP'] +\n",
+ " v['B_COST'] * v['CO_NO_TRIP']\n",
+ " )\n",
+ "\n",
+ " V_S_CAR = (\n",
+ " v['ASC_S_CAR'] +\n",
+ " v['B_TIME'] * v['TT_S_CAR'] +\n",
+ " v['B_COST'] * v['CO_S_CAR']\n",
+ " )\n",
+ "\n",
+ " V_TRANSIT = (\n",
+ " v['ASC_TRANSIT'] +\n",
+ " v['B_TIME'] * v['TT_TRANSIT'] +\n",
+ " v['B_COST'] * v['CO_TRANSIT']\n",
+ " )\n",
+ "\n",
+ " V_CAR = (\n",
+ " v['ASC_CAR'] +\n",
+ " v['B_TIME'] * v['TT_CAR'] +\n",
+ " v['B_COST'] * v['CO_CAR']\n",
+ " )\n",
+ "\n",
+ " V_S_MICRO = (\n",
+ " v['ASC_S_MICRO'] +\n",
+ " v['B_TIME'] * v['TT_S_MICRO'] +\n",
+ " v['B_COST'] * v['CO_S_MICRO']\n",
+ " )\n",
+ "\n",
+ " V_RIDEHAIL = (\n",
+ " v['ASC_RIDEHAIL'] +\n",
+ " v['B_TIME'] * v['TT_RIDEHAIL'] +\n",
+ " v['B_COST'] * v['CO_RIDEHAIL']\n",
+ " )\n",
+ "\n",
+ " V_WALK = (\n",
+ " v['ASC_WALK'] +\n",
+ " v['B_TIME'] * v['TT_WALK']\n",
+ " + v['B_COST'] * v['CO_WALK']\n",
+ " )\n",
+ "\n",
+ " V_UNKNOWN = (\n",
+ " v['ASC_UNKNOWN'] +\n",
+ " v['B_TIME'] * v['TT_UNKNOWN'] +\n",
+ " v['B_COST'] * v['CO_UNKNOWN']\n",
+ " )\n",
+ " \n",
+ " # Remember to exclude the input argument.\n",
+ " return {k:v for k,v in locals().items() if not k.startswith('_') and k != 'v'}\n",
+ "\n",
+ "\n",
+ "def get_utility_mapping(var: dict):\n",
+ " # Map alterative to utility functions.\n",
+ " return {\n",
+ " 1: var['V_P_MICRO'], \n",
+ " 2: var['V_NO_TRIP'],\n",
+ " 3: var['V_S_CAR'], \n",
+ " 4: var['V_TRANSIT'],\n",
+ " 5: var['V_CAR'], \n",
+ " 6: var['V_S_MICRO'],\n",
+ " 7: var['V_RIDEHAIL'], \n",
+ " 8: var['V_WALK'], \n",
+ " 9: var['V_UNKNOWN']\n",
+ " }\n",
+ "\n",
+ "\n",
+ "def get_availability_mapping(var: dict):\n",
+ " return {\n",
+ " 1: var['AV_P_MICRO'],\n",
+ " 2: var['AV_NO_TRIP'],\n",
+ " 3: var['AV_S_CAR'],\n",
+ " 4: var['AV_TRANSIT'],\n",
+ " 5: var['AV_CAR'],\n",
+ " 6: var['AV_S_MICRO'],\n",
+ " 7: var['AV_RIDEHAIL'],\n",
+ " 8: var['AV_WALK'],\n",
+ " 9: var['AV_UNKNOWN']\n",
+ " }"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# First, drop columns.\n",
+ "\n",
+ "train_data = drop_columns(train_data)\n",
+ "\n",
+ "# Next, scale time.\n",
+ "train_data = scale_time(train_data)\n",
+ "\n",
+ "# Scale cost.\n",
+ "train_data, scaler = scale_cost(train_data, SPLIT.TRAIN, None)\n",
+ "\n",
+ "# get dbs.\n",
+ "train_db = get_database(train_data, SPLIT.TRAIN)\n",
+ "\n",
+ "# get vars.\n",
+ "train_vars = get_variables()\n",
+ "\n",
+ "# exclude wrong points.\n",
+ "exclude_from_db(train_vars, train_db)\n",
+ "\n",
+ "train_params = get_params()\n",
+ "train_vars.update(train_params)\n",
+ "\n",
+ "train_V = get_utility_functions(train_vars)\n",
+ "train_vars.update(train_V)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "V = get_utility_mapping(train_vars)\n",
+ "av = get_availability_mapping(train_vars)\n",
+ "train_logprob = models.loglogit(V, av, train_vars['CHOICE'])\n",
+ "\n",
+ "model = bio.BIOGEME(train_db, train_logprob)\n",
+ "model.modelName = 'splitChoiceModel'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_results = model.estimate()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Results for model splitChoiceModel\n",
+ "Nbr of parameters:\t\t11\n",
+ "Sample size:\t\t\t129291\n",
+ "Excluded data:\t\t\t2133\n",
+ "Final log likelihood:\t\t-0.07647159\n",
+ "Akaike Information Criterion:\t22.15294\n",
+ "Bayesian Information Criterion:\t129.621\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(train_results.short_summary())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Value | \n",
+ " Rob. Std err | \n",
+ " Rob. t-test | \n",
+ " Rob. p-value | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | ASC_CAR | \n",
+ " 88.793727 | \n",
+ " 2.832838e-02 | \n",
+ " 3.134444e+03 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | ASC_NO_TRIP | \n",
+ " -623.288333 | \n",
+ " 1.797693e+308 | \n",
+ " -3.467156e-306 | \n",
+ " 1.0 | \n",
+ "
\n",
+ " \n",
+ " | ASC_P_MICRO | \n",
+ " 83.286468 | \n",
+ " 5.705798e-02 | \n",
+ " 1.459681e+03 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | ASC_RIDEHAIL | \n",
+ " 88.760204 | \n",
+ " 2.370587e-02 | \n",
+ " 3.744230e+03 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | ASC_S_CAR | \n",
+ " 88.806707 | \n",
+ " 2.278816e-02 | \n",
+ " 3.897055e+03 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | ASC_S_MICRO | \n",
+ " 85.102997 | \n",
+ " 3.619103e-02 | \n",
+ " 2.351494e+03 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | ASC_TRANSIT | \n",
+ " 85.580932 | \n",
+ " 5.083868e-02 | \n",
+ " 1.683382e+03 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | ASC_UNKNOWN | \n",
+ " 0.000000 | \n",
+ " 1.797693e+308 | \n",
+ " 0.000000e+00 | \n",
+ " 1.0 | \n",
+ "
\n",
+ " \n",
+ " | ASC_WALK | \n",
+ " 102.957297 | \n",
+ " 1.408207e-01 | \n",
+ " 7.311235e+02 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | B_COST | \n",
+ " -2879.406137 | \n",
+ " 2.303798e+01 | \n",
+ " -1.249852e+02 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | B_TIME | \n",
+ " -107.199308 | \n",
+ " 2.970404e-01 | \n",
+ " -3.608913e+02 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " utility_p_micro | \n",
+ " utility_no_trip | \n",
+ " utility_s_car | \n",
+ " utility_transit | \n",
+ " utility_car | \n",
+ " utility_s_micro | \n",
+ " utility_ridehail | \n",
+ " utility_walk | \n",
+ " utility_unknown | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 67.411217 | \n",
+ " -623.288333 | \n",
+ " 63.828875 | \n",
+ " 64.734191 | \n",
+ " 63.815895 | \n",
+ " 64.959539 | \n",
+ " 82.764109 | \n",
+ " 44.675225 | \n",
+ " -21.398065 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " -91.860937 | \n",
+ " -623.288333 | \n",
+ " -71.415959 | \n",
+ " -97.629531 | \n",
+ " -71.428939 | \n",
+ " -188.098667 | \n",
+ " -48.301289 | \n",
+ " -669.366009 | \n",
+ " -119.831550 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 58.227962 | \n",
+ " -623.288333 | \n",
+ " 75.253692 | \n",
+ " 55.372684 | \n",
+ " 56.017998 | \n",
+ " 50.368803 | \n",
+ " 55.984476 | \n",
+ " 3.505302 | \n",
+ " -27.073506 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 48.651631 | \n",
+ " -623.288333 | \n",
+ " 47.899304 | \n",
+ " 45.610470 | \n",
+ " 47.886323 | \n",
+ " 35.153530 | \n",
+ " 67.326805 | \n",
+ " -39.426846 | \n",
+ " -32.991877 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 71.481893 | \n",
+ " -623.288333 | \n",
+ " 67.285461 | \n",
+ " 68.883882 | \n",
+ " 67.272481 | \n",
+ " 71.427198 | \n",
+ " 86.113878 | \n",
+ " 62.924686 | \n",
+ " -18.882302 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " utility_p_micro | \n",
+ " utility_no_trip | \n",
+ " utility_s_car | \n",
+ " utility_transit | \n",
+ " utility_car | \n",
+ " utility_s_micro | \n",
+ " utility_ridehail | \n",
+ " utility_walk | \n",
+ " utility_unknown | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 2.149431e-07 | \n",
+ " 2.319104e-307 | \n",
+ " 5.977672e-09 | \n",
+ " 1.478107e-08 | \n",
+ " 5.900582e-09 | \n",
+ " 1.851711e-08 | \n",
+ " 9.999997e-01 | \n",
+ " 2.872153e-17 | \n",
+ " 5.793521e-46 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 1.208607e-19 | \n",
+ " 1.933302e-250 | \n",
+ " 9.150116e-11 | \n",
+ " 3.775868e-22 | \n",
+ " 9.032113e-11 | \n",
+ " 1.935398e-61 | \n",
+ " 1.000000e+00 | \n",
+ " 1.883732e-270 | \n",
+ " 8.606027e-32 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 4.034777e-08 | \n",
+ " 4.236948e-304 | \n",
+ " 9.999999e-01 | \n",
+ " 2.321603e-09 | \n",
+ " 4.426337e-09 | \n",
+ " 1.558225e-11 | \n",
+ " 4.280415e-09 | \n",
+ " 6.919424e-32 | \n",
+ " 3.629633e-45 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 7.753087e-09 | \n",
+ " 1.173968e-300 | \n",
+ " 3.653787e-09 | \n",
+ " 3.704379e-10 | \n",
+ " 3.606667e-09 | \n",
+ " 1.064937e-14 | \n",
+ " 1.000000e+00 | \n",
+ " 4.339885e-47 | \n",
+ " 2.704892e-44 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 4.419870e-07 | \n",
+ " 8.138306e-309 | \n",
+ " 6.651541e-09 | \n",
+ " 3.289330e-08 | \n",
+ " 6.565760e-09 | \n",
+ " 4.184619e-07 | \n",
+ " 9.999991e-01 | \n",
+ " 8.493006e-11 | \n",
+ " 2.516158e-46 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "