classification_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification Analysis\n",
    "This notebook will contain classification analysis for both the sensed and pipelined algorithms. Analysis will be preformed in regards for the sensed and pipelined algorthms themselves, as well as the ensemble algorithms. The analysis for the ensemble algorithm will focus on the HAMF android phones and the HAHF iOS phones."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for reading and validating data\n",
    "import emeval.input.spec_details as eisd\n",
    "import emeval.input.phone_view as eipv\n",
    "import emeval.input.eval_view as eiev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import emeval.viz.phone_view as ezpv\n",
    "import emeval.viz.eval_view as ezev\n",
    "import emeval.viz.geojson as ezgj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for analysized view\n",
    "import emeval.analysed.phone_view as eapv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import emeval.metrics.segmentation as ems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Metrics helpers\n",
    "import emeval.metrics.dist_calculations as emd\n",
    "import emeval.metrics.reference_trajectory as emr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.options.display.float_format = '{:.6f}'.format\n",
    "import arrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "import scipy.interpolate\n",
    "import scipy.integrate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For plots\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For maps\n",
    "import geopandas as gpd\n",
    "import shapely as shp\n",
    "import folium\n",
    "import branca.element as bre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For easier debugging while working on modules\n",
    "import importlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import arrow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load in Phone Views from the file spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASTORE_LOC = \"bin/data\"\n",
    "AUTHOR_EMAIL = \"shankari@eecs.berkeley.edu\"\n",
    "sd_la = eisd.FileSpecDetails(DATASTORE_LOC, AUTHOR_EMAIL, \"unimodal_trip_car_bike_mtv_la\")\n",
    "sd_sj = eisd.FileSpecDetails(DATASTORE_LOC, AUTHOR_EMAIL, \"car_scooter_brex_san_jose\")\n",
    "sd_ucb = eisd.FileSpecDetails(DATASTORE_LOC, AUTHOR_EMAIL, \"train_bus_ebike_mtv_ucb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_la = eipv.PhoneView(sd_la)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_sj = eipv.PhoneView(sd_sj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pv_ucb = eipv.PhoneView(sd_ucb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get sensed data for each trip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "ems.fill_sensed_section_ranges(pv_la)\n",
    "ems.fill_sensed_section_ranges(pv_sj)\n",
    "ems.fill_sensed_section_ranges(pv_ucb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get sensed and ground truth temporal histories (timelines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_trip_ss_and_gts_timeline(pv, os, role):\n",
    "    assert os in ['android', 'ios'], 'UNKNOWN OS'\n",
    "    assert role in ['accuracy_control', 'HAHFDC', 'HAMFDC', 'MAHFDC', 'power_control'], \"UNKNOWN ROLE\"\n",
    "    trips = []\n",
    "    for phone_os, phone_map in pv.map().items():\n",
    "        if os != phone_os:\n",
    "            continue\n",
    "        for phone_label, phone_detail_map in phone_map.items():\n",
    "            if \"control\" in phone_detail_map[\"role\"]:\n",
    "                continue\n",
    "            for r in phone_detail_map[\"evaluation_ranges\"]:\n",
    "                if r['eval_role_base'] != role:\n",
    "                    continue\n",
    "                tr_ss  = []\n",
    "                tr_gts = []\n",
    "                for i, tr in enumerate(r[\"evaluation_trip_ranges\"]):\n",
    "                    for ss in tr[\"sensed_section_ranges\"]:\n",
    "                        ## append the sensed section data\n",
    "                        tr_ss.append(ss)\n",
    "                    for section in tr[\"evaluation_section_ranges\"]:\n",
    "                        ## get the ground truth section data\n",
    "                        section_gt_leg = pv.spec_details.get_ground_truth_for_leg(tr['trip_id_base'],\n",
    "                                                                                  section['trip_id_base'],\n",
    "                                                                                  tr['start_ts'],\n",
    "                                                                                  tr['end_ts'])\n",
    "                        \n",
    "                        if section_gt_leg[\"type\"] == \"WAITING\": \n",
    "                            continue\n",
    "                        gts = {'start_ts': section['start_ts'], \n",
    "                               'end_ts': section['end_ts'], \n",
    "                               'mode': section_gt_leg['mode']}\n",
    "                        tr_gts.append(gts)\n",
    "                # now, we build a timeline for each trip\n",
    "                trip = tr.copy()\n",
    "                trip['ss_timeline']  = tr_ss\n",
    "                trip['gts_timeline'] = tr_gts\n",
    "                trips.append(trip)\n",
    "    return trips"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the Base Mode Maps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### raw base mode map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RBMM = {\n",
    "    \"WALKING\": \"WALKING\",\n",
    "    \"RUNNING\" : \"WALKING\", \n",
    "    \"CYCLING\" : \"CYCLING\",\n",
    "    \"BICYCLING\": \"CYCLING\",\n",
    "    \"E_BIKE\" : \"CYCLING\",\n",
    "    \"ESCOOTER\": \"CYCLING\", \n",
    "    \"AUTOMOTIVE\" : \"AUTOMOTIVE\",\n",
    "    \"BUS\": \"AUTOMOTIVE\",\n",
    "    \"TRAIN\": \"AUTOMOTIVE\",\n",
    "    \"LIGHT_RAIL\": \"AUTOMOTIVE\",\n",
    "    \"SUBWAY\": \"AUTOMOTIVE\",\n",
    "    \"CAR\": \"AUTOMOTIVE\",\n",
    "    \"AIR_OR_HSR\": \"AIR_OR_HSR\",\n",
    "    \"MISALIGNED\" : \"MISALIGNED\", \n",
    "    \"NO_SENSED\" : \"MISALIGNED\", \n",
    "    \"NO_GT\" : \"MISALIGNED\", \n",
    "    \"INVALID\" : \"UNKNOWN\", \n",
    "    \"UNKNOWN\" : \"UNKNOWN\", \n",
    "    'NO_SENSED_START' : 'NO_SENSED',\n",
    "    'NO_SENSED_MIDDLE' : 'NO_SENSED',\n",
    "    'NO_SENSED_END' : 'NO_SENSED',\n",
    "    'NO_GT_START' : 'NO_GT',\n",
    "    'NO_GT_MIDDLE' : 'NO_GT',\n",
    "    'NO_GT_END' : 'NO_GT'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### cleaned base mode map\n",
    "\n",
    "e-mission-server.emission.core.wrapper.motionactivity.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CBMM = {\n",
    "    0 : 'AUTOMOTIVE', \n",
    "    1 : 'CYCLING', \n",
    "    2 : 'WALKING', \n",
    "    3 : 'WALKING', \n",
    "    4 : 'UNKNOWN', \n",
    "    5 : 'WALKING', \n",
    "    7 : 'WALKING', \n",
    "    8 : 'WALKING', \n",
    "    9 : 'UNKNOWN', \n",
    "    10 : 'AUTOMOTIVE', \n",
    "    11 : 'AIR_OR_HSR', \n",
    "    \"WALKING\": \"WALKING\",\n",
    "    \"RUNNING\" : \"WALKING\", \n",
    "    \"CYCLING\" : \"CYCLING\",\n",
    "    \"BICYCLING\": \"CYCLING\",\n",
    "    \"E_BIKE\" : \"CYCLING\",\n",
    "    \"ESCOOTER\": \"CYCLING\", \n",
    "    \"AUTOMOTIVE\" : \"AUTOMOTIVE\",\n",
    "    \"BUS\": \"AUTOMOTIVE\",\n",
    "    \"TRAIN\": \"AUTOMOTIVE\",\n",
    "    \"LIGHT_RAIL\": \"AUTOMOTIVE\",\n",
    "    \"SUBWAY\": \"AUTOMOTIVE\",\n",
    "    \"CAR\": \"AUTOMOTIVE\",\n",
    "    \"AIR_OR_HSR\": \"AIR_OR_HSR\", \n",
    "    \"NO_SENSED\" : \"MISALIGNED\", \n",
    "    \"NO_GT\" : \"MISALIGNED\",\n",
    "    \"MISALIGNED\" : \"MISALIGNED\", \n",
    "    \"UNKNOWN\" : \"UNKNOWN\",\n",
    "    'NO_SENSED_START' : 'NO_SENSED',\n",
    "    'NO_SENSED_MIDDLE' : 'NO_SENSED',\n",
    "    'NO_SENSED_END' : 'NO_SENSED',\n",
    "    'NO_GT_START' : 'NO_GT',\n",
    "    'NO_GT_MIDDLE' : 'NO_GT',\n",
    "    'NO_GT_END' : 'NO_GT'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### inferred base mode maps\n",
    "\n",
    "e-mission-server.emission.core.wrapper.modeprediction.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### random forest base mode map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RFBMM = {0 : 'UNKNOWN',    # UNKNOWN\n",
    "        1 : 'WALKING',    # WALKING\n",
    "        2 : 'CYCLING',    # BICYCLING\n",
    "        3 : 'BUS',        # BUS\n",
    "        4 : 'TRAIN',      # TRAIN\n",
    "        5 : 'CAR',        # CAR\n",
    "        6 : 'AIR_OR_HSR', # AIR_OR_HSR\n",
    "        7 : 'TRAIN',      # SUBWAY\n",
    "        8 : 'TRAIN',      # TRAM\n",
    "        9 : 'TRAIN',      # LIGHT_RAIL\n",
    "         \"WALKING\": \"WALKING\",\n",
    "         \"CYCLING\" : \"CYCLING\",\n",
    "         \"BICYCLING\": \"CYCLING\",\n",
    "         \"E_BIKE\" : \"CYCLING\",\n",
    "         \"ESCOOTER\": \"CYCLING\", \n",
    "         \"BUS\": \"BUS\",\n",
    "         \"TRAIN\": \"TRAIN\",\n",
    "         \"LIGHT_RAIL\": \"TRAIN\",\n",
    "         \"SUBWAY\": \"TRAIN\",\n",
    "         \"CAR\": \"CAR\",\n",
    "         \"AIR_OR_HSR\": \"AIR_OR_HSR\",\n",
    "         \"UNKNOWN\" : \"UNKNOWN\",\n",
    "         \"NO_SENSED\" : \"MISALIGNED\", \n",
    "         \"NO_GT\" : \"MISALIGNED\",\n",
    "         \"MISALIGNED\" : \"MISALIGNED\", \n",
    "    'NO_SENSED_START' : 'NO_SENSED',\n",
    "    'NO_SENSED_MIDDLE' : 'NO_SENSED',\n",
    "    'NO_SENSED_END' : 'NO_SENSED',\n",
    "    'NO_GT_START' : 'NO_GT',\n",
    "    'NO_GT_MIDDLE' : 'NO_GT',\n",
    "    'NO_GT_END' : 'NO_GT'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### rule+GIS base mode map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GISBMM = {0 : 'UNKNOWN',    # UNKNOWN\n",
    "        1 : 'WALKING',    # WALKING\n",
    "        2 : 'CYCLING',    # BICYCLING\n",
    "        3 : 'BUS',        # BUS\n",
    "        4 : 'TRAIN',      # TRAIN\n",
    "        5 : 'CAR',        # CAR\n",
    "        6 : 'AIR_OR_HSR', # AIR_OR_HSR\n",
    "        7 : 'SUBWAY',      # SUBWAY\n",
    "        8 : 'TRAIN',      # TRAM\n",
    "        9 : 'TRAIN',      # LIGHT_RAIL\n",
    "         \"WALKING\": \"WALKING\",\n",
    "         \"CYCLING\" : \"CYCLING\",\n",
    "         \"BICYCLING\": \"CYCLING\",\n",
    "        \"E_BIKE\" : \"CYCLING\",\n",
    "         \"ESCOOTER\": \"CYCLING\", \n",
    "         \"BUS\": \"BUS\",\n",
    "         \"TRAIN\": \"TRAIN\",\n",
    "         \"LIGHT_RAIL\": \"TRAIN\",\n",
    "         \"SUBWAY\": \"TRAIN\",\n",
    "         \"CAR\": \"CAR\",\n",
    "         \"AIR_OR_HSR\": \"AIR_OR_HSR\",\n",
    "         \"UNKNOWN\" : \"UNKNOWN\",\n",
    "         \"NO_SENSED\" : \"MISALIGNED\", \n",
    "         \"NO_GT\" : \"MISALIGNED\",\n",
    "         \"MISALIGNED\" : \"MISALIGNED\", \n",
    "    'NO_SENSED_START' : 'NO_SENSED',\n",
    "    'NO_SENSED_MIDDLE' : 'NO_SENSED',\n",
    "    'NO_SENSED_END' : 'NO_SENSED',\n",
    "    'NO_GT_START' : 'NO_GT',\n",
    "    'NO_GT_MIDDLE' : 'NO_GT',\n",
    "    'NO_GT_END' : 'NO_GT'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pad the start at end of the timelines for a given trip, while also filling in gaps in the middle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_timelines(trip):\n",
    "    ss_timeline = trip['ss_timeline']\n",
    "    gt_timeline = trip['gts_timeline']\n",
    "    ss_aligned_timeline = []\n",
    "    gt_aligned_timeline = []\n",
    "    ####### FILL IN SENSED TIMELINE #######\n",
    "    ### fill in start ###\n",
    "    if len(ss_timeline) == 0:\n",
    "        if len(gt_timeline) == 0:\n",
    "            return ss_timeline, gt_timeline\n",
    "        else:\n",
    "            ss_timeline.append(\n",
    "                {\n",
    "                    'mode' : 'NO_SENSED_START',\n",
    "                    'start_ts' : gt_timeline[0]['start_ts'],\n",
    "                    'end_ts' : gt_timeline[-1]['end_ts']\n",
    "                }\n",
    "            )\n",
    "    if len(gt_timeline) == 0:\n",
    "        gt_timeline.append(\n",
    "            {\n",
    "                'mode' : 'NO_GT_START',\n",
    "                'start_ts' : ss_timeline[0]['start_ts'],\n",
    "                'end_ts' : ss_timeline[-1]['end_ts']\n",
    "            }\n",
    "        )\n",
    "    if 'data' in ss_timeline[0]:\n",
    "        start_misalignment = ss_timeline[0]['data']['start_ts'] - gt_timeline[0]['start_ts']\n",
    "        end_misalignment = ss_timeline[-1]['data']['end_ts'] - gt_timeline[-1]['end_ts']\n",
    "    else:\n",
    "        start_misalignment = ss_timeline[0]['start_ts'] - gt_timeline[0]['start_ts']\n",
    "        end_misalignment = ss_timeline[-1]['end_ts'] - gt_timeline[-1]['end_ts']\n",
    "\n",
    "    if start_misalignment > 0:\n",
    "        if 'data' in ss_timeline[0].keys():\n",
    "            ss_timeline[0] = ss_timeline[0]['data']\n",
    "        \n",
    "        ss_aligned_timeline.append(\n",
    "            {\n",
    "                'mode' : 'NO_SENSED_START',\n",
    "                'start_ts' : ss_timeline[0]['start_ts'] - start_misalignment,\n",
    "                'end_ts' : ss_timeline[0]['start_ts']\n",
    "            }\n",
    "        )\n",
    "    ### fill in meat ###\n",
    "    for ss in ss_timeline:\n",
    "        if 'data' in ss.keys():\n",
    "            ss = ss['data']\n",
    "        if 'sensed_mode' in ss.keys():\n",
    "            ss['mode'] = ss['sensed_mode']\n",
    "        if len(ss_aligned_timeline) > 0:\n",
    "            ## check to see if there is a gap ##\n",
    "            if ss['start_ts'] - ss_aligned_timeline[-1]['end_ts'] > 0:\n",
    "                ## fill in the blank\n",
    "                ss_aligned_timeline.append(\n",
    "                    {\n",
    "                        'mode' : 'NO_SENSED_MIDDLE', \n",
    "                        'start_ts' : ss_aligned_timeline[-1]['end_ts'],\n",
    "                        'end_ts' : ss['start_ts']\n",
    "                    }\n",
    "                )\n",
    "            \n",
    "        ## the timeline is continuous, and we can fill our section ##\n",
    "        ss_aligned_timeline.append(ss)\n",
    "    ### fill in end ###\n",
    "    if end_misalignment < 0:\n",
    "        ss = ss_timeline[-1]\n",
    "        if 'data' in ss.keys():\n",
    "            ss = ss['data']\n",
    "        ss_aligned_timeline.append(\n",
    "            {\n",
    "                'mode' : 'NO_SENSED_END',\n",
    "                'start_ts' : ss['end_ts'],\n",
    "                'end_ts' : ss['end_ts'] - end_misalignment\n",
    "            }\n",
    "        )\n",
    "    ####### FILL IN GT TIMELINE #######\n",
    "    ### fill in start ###\n",
    "    if start_misalignment < 0:\n",
    "        gt_aligned_timeline.append(\n",
    "            {\n",
    "                'mode' : 'NO_GT_START',\n",
    "                'start_ts' : gt_timeline[0]['start_ts'] + start_misalignment,\n",
    "                'end_ts' : gt_timeline[0]['start_ts']\n",
    "            }\n",
    "        )\n",
    "    ### fill in meat ###\n",
    "    for gts in gt_timeline:\n",
    "        if len(gt_aligned_timeline) > 0:\n",
    "            ## fill in the blank ##\n",
    "            if gts['start_ts'] - gt_aligned_timeline[-1]['end_ts'] > 0:\n",
    "                gt_aligned_timeline.append(\n",
    "                    {\n",
    "                        'mode' : 'NO_GT_MIDDLE',\n",
    "                        'start_ts' : gt_aligned_timeline[-1]['end_ts'],\n",
    "                        'end_ts' : gts['start_ts']\n",
    "                    }\n",
    "                )\n",
    "        gt_aligned_timeline.append(gts)\n",
    "    ### fill in end ###\n",
    "    if end_misalignment > 0:\n",
    "        gt_aligned_timeline.append(\n",
    "            {\n",
    "                'mode' : 'NO_GT_END',\n",
    "                'start_ts' : gt_aligned_timeline[-1]['end_ts'],\n",
    "                'end_ts' : gt_aligned_timeline[-1]['end_ts'] + end_misalignment\n",
    "            }\n",
    "        )\n",
    "    return ss_aligned_timeline, gt_aligned_timeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get the classification metrics (true/false positive, true/false negative) for each Base Mode for a given trip/set-of-trips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_binary_class_in_sec(os, role, pv, BASE_MODE, test=False, test_trip=None):\n",
    "    if not test:\n",
    "        if type(pv) is not list: pv = [pv]\n",
    "        trips = []\n",
    "        for v in pv:\n",
    "            trips.extend(get_trip_ss_and_gts_timeline(v, os, role))\n",
    "    else:\n",
    "        trips = test_trip if type(test_trip) is list else [test_trip]\n",
    "    TP, FN, FP, TN = {}, {}, {}, {}\n",
    "    for trip in trips:\n",
    "        ss_timeline, gt_timeline = align_timelines(trip)\n",
    "        for mode in set(BASE_MODE.values()):\n",
    "            for ss in ss_timeline:\n",
    "                for gts in gt_timeline:\n",
    "                    if ss['end_ts'] >= gts['start_ts'] and ss['start_ts'] <= gts['end_ts']:\n",
    "                        dur = min(ss['end_ts'], gts['end_ts']) - max(ss['start_ts'], gts['start_ts'])\n",
    "                        if BASE_MODE[mode] == BASE_MODE[ss['mode']] and BASE_MODE[mode] == BASE_MODE[gts['mode']]:\n",
    "                            TP[mode] = TP.setdefault(mode, 0) + dur\n",
    "                        elif BASE_MODE[mode] == BASE_MODE[ss['mode']] and BASE_MODE[mode] != BASE_MODE[gts['mode']]:\n",
    "                            FP[mode] = FP.setdefault(mode, 0) + dur\n",
    "                        elif BASE_MODE[mode] != BASE_MODE[ss['mode']] and BASE_MODE[mode] == BASE_MODE[gts['mode']]:\n",
    "                            FN[mode] = FN.setdefault(mode, 0) + dur\n",
    "                        else:\n",
    "                            TN[mode] = TN.setdefault(mode, 0) + dur\n",
    "                    \n",
    "    return TP, FP, FN, TN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# $F_\\beta$ score\n",
    "$$\n",
    "F_\\beta = \\frac {(1 + \\beta^2) \\cdot \\mathrm{true\\ positive} }{(1 + \\beta^2) \\cdot \\mathrm{true\\ positive} + \\beta^2 \\cdot \\mathrm{false\\ negative} + \\mathrm{false\\ positive}}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_F_score(os, role, pv, BASE_MODE, beta=1, test=False, test_trip=None):\n",
    "    if not test:\n",
    "        assert os in ['android', 'ios'], 'UNKNOWN OS'\n",
    "        assert role in ['accuracy_control', 'HAHFDC', 'HAMFDC', 'MAHFDC', 'power_control'], \"UNKNOWN ROLE\"\n",
    "        (TP, FP, FN, TN) = get_binary_class_in_sec(os, role, pv, BASE_MODE)\n",
    "    else:\n",
    "        (TP, FP, FN, TN) = get_binary_class_in_sec(os, role, pv, BASE_MODE, test=True, test_trip=test_trip)\n",
    "    F_score = {}\n",
    "    for mode in set(BASE_MODE.values()):\n",
    "        numerator   = (1 + beta**2) * TP.setdefault(mode, 0)\n",
    "        denominator = (1+beta**2) * TP.setdefault(mode, 0) + beta**2*FN.setdefault(mode, 0) + FP.setdefault(mode, 0)\n",
    "        try:\n",
    "            F_score[mode] = (numerator)/(denominator)\n",
    "        except:\n",
    "            F_score[mode] = np.nan\n",
    "    # initializing K \n",
    "    K = 10\n",
    "    for key in F_score:\n",
    "\n",
    "        # rounding to K using round()\n",
    "        F_score[key] = round(F_score[key], K)\n",
    "    return F_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get the support for each base mode in a set of trips, which is the sum of confusion matrix row sums for each mode that maps to a base mode, $M_{bm} = \\{ m : b(m) = bm, m \\in M \\}$ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_support(os, role, pv, BASE_MODE):\n",
    "    if type(pv) is not list: pv = [pv]\n",
    "    trips = []\n",
    "    for v in pv:\n",
    "        trips.extend(get_trip_ss_and_gts_timeline(v, os, role))\n",
    "    support = {}\n",
    "    for trip in trips:\n",
    "        ## get gts dur\n",
    "        gt_dur = 0\n",
    "        for gts in trip['gts_timeline']:\n",
    "            mode = BASE_MODE[gts['mode']]\n",
    "            support[mode] = support.setdefault(mode, 0) + gts['end_ts'] - gts['start_ts']\n",
    "            gt_dur += (gts['end_ts'] - gts['start_ts'])\n",
    "        ## check if there is a NO_GT mode\n",
    "        ss_dur = 0\n",
    "        for ss in trip['ss_timeline']:\n",
    "            try:\n",
    "                ss_dur += (ss['end_ts'] - ss['start_ts'])\n",
    "            except:\n",
    "                ss_dur += (ss['data']['end_ts'] - ss['data']['start_ts'])\n",
    "        support['NO_GT'] = support.setdefault('NO_GT', 0) + max(0, ss_dur - gt_dur)\n",
    "    return support"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weighted $F_1$ Score\n",
    "\n",
    "\\begin{equation}\\label{eq:w_f_score}\n",
    "    F_1^{avg} = \\sum_{i=1}^{|BM|} \\left[  \\sum_{j=1} ^ {|M_{bm_i}|} \\sum_{k = 1}^{|M^{inf}|} cm_{j, k} \\right] \\cdot F_1^{bm}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_f_score(os, role, pv, BASE_MODE):\n",
    "    support = get_support(os, role, pv, BASE_MODE)\n",
    "    total_support = sum(support.values())\n",
    "    F_scores = get_F_score(os, role, pv, BASE_MODE)\n",
    "    weighted_f_score = sum(\n",
    "        support[mode]/total_support * F_scores.setdefault(mode, 0) \n",
    "        for mode in support.keys()\n",
    "        if not np.isnan(F_scores.setdefault(mode, 0))\n",
    "    )\n",
    "    return weighted_f_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_f_scores(os):\n",
    "    fig, ax = plt.subplots(1,3, figsize = (15,5), dpi=300, sharey=True, sharex=True)\n",
    "    for i, role in enumerate(['HAHFDC', 'HAMFDC', 'MAHFDC']):\n",
    "        raw   = get_F_score(os, role, [pv_la, pv_sj, pv_ucb], RBMM)\n",
    "        m_clean = get_F_score(os, role, [mcv_la, mcv_sj, mcv_ucb], CBMM)\n",
    "        g_clean = get_F_score(os, role, [gcv_la, gcv_sj, gcv_ucb], CBMM)\n",
    "        rf    = get_F_score(os, role, [rfv_la, rfv_sj, rfv_ucb], RFBMM)\n",
    "        gis   = get_F_score(os, role, [gisv_la, gisv_sj, gisv_ucb], GISBMM)\n",
    "        df = pd.DataFrame(\n",
    "            [raw, m_clean, g_clean, rf, gis], \n",
    "            index = ['raw', 'master clean', 'GIS clean', 'random forest', 'GIS'], \n",
    "            columns=['WALKING', 'CYCLING', 'AUTOMOTIVE', 'CAR', 'BUS', 'TRAIN', 'SUBWAY']\n",
    "        )\n",
    "        \n",
    "        df.T.plot(style='o', ax=ax[i], title=f' {role} ').legend(loc='lower left')\n",
    "        ax[i].set_xticks(range(len(df.T)))\n",
    "        ax[i].set_xticklabels(df.columns, rotation = 80)\n",
    "    title = f\"$F_1$ Scores by Base Mode for Phones Running {os} at Various Configuration Settings\"\n",
    "    plt.suptitle(title, weight='bold', size='x-large')\n",
    "    fig.savefig(f\"images/f_scores_for_{os}\",  bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot $F$ scores for android/ios on select configuration settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_f_scores_selected():\n",
    "    fig, ax = plt.subplots(1,2, figsize = (10,3), dpi=300, sharey=True, sharex=True)\n",
    "    for i, (os, role) in enumerate([['android', 'HAMFDC'], ['ios', 'HAHFDC']]):\n",
    "        raw   = get_F_score(os, role, [pv_la, pv_sj, pv_ucb], RBMM)\n",
    "        m_clean = get_F_score(os, role, [mcv_la, mcv_sj, mcv_ucb], CBMM)\n",
    "        g_clean = get_F_score(os, role, [gcv_la, gcv_sj, gcv_ucb], CBMM)\n",
    "        rf    = get_F_score(os, role, [rfv_la, rfv_sj, rfv_ucb], RFBMM)\n",
    "        gis   = get_F_score(os, role, [gisv_la, gisv_sj, gisv_ucb], GISBMM)\n",
    "        df = pd.DataFrame(\n",
    "            [raw, m_clean, g_clean, rf, gis], \n",
    "            index = ['raw', 'master clean', 'GIS clean', 'random forest', 'GIS'], \n",
    "            columns=['WALKING', 'CYCLING', 'AUTOMOTIVE', 'CAR', 'BUS', 'TRAIN', 'SUBWAY']\n",
    "        )\n",
    "        \n",
    "        df.T.plot(style='o', ax=ax[i], title=f'$F_1$ Scores by Base Mode \\n{os}:{role} ').legend(loc='lower left')\n",
    "        ax[i].set_xticks(range(len(df.T)))\n",
    "        ax[i].set_xticklabels(df.columns, rotation = 80)\n",
    "#     title = f\"$F_1$ Scores by Base Mode for Selected OS Setting Configurations\"\n",
    "#     plt.suptitle(title, weight='bold', size='x-large')\n",
    "    fig.savefig(f\"images/f_scores_selected\",  bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix\n",
    "We will now generate confusion matrices based off OS and role, with the acctual modes as the rows, the predicted modes as the columns, and the entries as the base unit for the duration measurement"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### cleaned index map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CIM = {0 : 'IN_VEHICLE', \n",
    "       1 : 'BICYCLING', \n",
    "       2 : 'ON_FOOT', \n",
    "       3 : 'STILL', \n",
    "       4 : 'UNKNOWN', \n",
    "       5 : 'TILTING', \n",
    "       7 : 'WALKING', \n",
    "       8 : 'RUNNING', \n",
    "       9 : 'NONE', \n",
    "       10 : 'STOPPED_WHILE_IN_VEHICLE', \n",
    "       11 : 'AIR_OR_HSR'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### inferred index map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IIM = {\n",
    "    0 : 'UNKNOWN', \n",
    "    1 : 'WALKING', \n",
    "    2 : 'BICYCLING', \n",
    "    3 : 'BUS', \n",
    "    4 : 'TRAIN', \n",
    "    5 : 'CAR', \n",
    "    6 : 'AIR_OR_HSR',\n",
    "    7 : 'SUBWAY',\n",
    "    8 : 'TRAM',\n",
    "    9 : 'LIGHT_RAIL'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_confusion_matrix(os, role, pv, test=False, test_trip=None):\n",
    "    cm_l = []\n",
    "    if not test:\n",
    "        assert os in ['android', 'ios'], 'UNKNOWN OS'\n",
    "        assert role in ['accuracy_control', 'HAHFDC', 'HAMFDC', 'MAHFDC', 'power_control'], \"UNKNOWN ROLE\"\n",
    "        if type(pv) is not list:\n",
    "            pv = [pv]\n",
    "        trips = []\n",
    "        for v in pv :\n",
    "            trips.extend(get_trip_ss_and_gts_timeline(v, os, role))\n",
    "    else:\n",
    "        trips = test_trip if type(test_trip) is list else [test_trip]\n",
    "    for trip in trips:\n",
    "        ss_timeline, gt_timeline = align_timelines(trip)\n",
    "        for ss in ss_timeline:\n",
    "            cm = {}\n",
    "            for gts in gt_timeline:\n",
    "                if ss['end_ts'] >= gts['start_ts'] and ss['start_ts'] <= gts['end_ts']:\n",
    "                    dur = min(ss['end_ts'], gts['end_ts']) - max(ss['start_ts'], gts['start_ts'])\n",
    "                    cm[gts['mode']] = cm.setdefault(gts['mode'], 0) + dur\n",
    "            cm['sensed_mode'] = ss['mode']\n",
    "            \n",
    "            cm_l.append(cm)\n",
    "    return cm_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_dict = {\n",
    "    'WALKING' : 0,\n",
    "    'RUNNING' : 1,\n",
    "    'ON_FOOT' : 2,\n",
    "    'CYCLING' : 3,\n",
    "    'BICYCLING' : 4,\n",
    "    'AUTOMOTIVE' : 5,\n",
    "    'IN_VEHICLE' : 6,\n",
    "    'CAR': 7,\n",
    "    'BUS': 8,\n",
    "    'SUBWAY' : 9,\n",
    "    'LIGHT_RAIL' : 10,\n",
    "    'TRAIN' : 11,\n",
    "    'AIR_OR_HSR' : 12,\n",
    "    'INVALID' : 13,\n",
    "    'UNKNOWN' : 14,\n",
    "    'NO_SENSED_START' : 15,\n",
    "    'NO_SENSED_MIDDLE' : 16,\n",
    "    'NO_SENSED_END' : 17\n",
    "}\n",
    "def sort_key(s):\n",
    "    return sort_dict[s]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_cm(os, pv, d_type, INDEX_MAP=None):\n",
    "    fig, ax = plt.subplots(1,3, figsize=(22,10), dpi=300, sharey=True)\n",
    "    y=.95\n",
    "    fig.text(0.5, 0.0, 'Predicted Label', ha='center', fontsize='xx-large')\n",
    "    fig.text(0.04, 0.5, 'True Label', va='center', rotation='vertical', fontsize='xx-large')\n",
    "    for k, role in enumerate([\"HAHFDC\", \"HAMFDC\", \"MAHFDC\"]):\n",
    "        if d_type =='raw':\n",
    "            title = f\"Confusion Matrices for Raw Output Data on Phones Running {os} \\n by Calibration Settings\"\n",
    "            df = pd.DataFrame(get_confusion_matrix(os, role, pv)).groupby('sensed_mode').sum()\n",
    "            df = pd.DataFrame(df, index=sorted(df.index, key=sort_key))\n",
    "            fname = f\"images/raw_cm_{os}\"\n",
    "        elif d_type == 'clean':\n",
    "            title = f\"Confusion Matrices for Clean Output Data on Phones Running {os} \\n by Calibration Settings\"\n",
    "            df = pd.DataFrame(get_confusion_matrix(os, role, pv)).groupby('sensed_mode').sum().rename(index=INDEX_MAP)\n",
    "            df = pd.DataFrame(df, index=sorted(df.index, key=sort_key))\n",
    "            fname = f\"images/clean_cm_{os}\"\n",
    "        elif d_type == 'random_forest' or 'gis':\n",
    "            df = pd.DataFrame(get_confusion_matrix(os, role, pv)).groupby('sensed_mode').sum().rename(index=INDEX_MAP)\n",
    "            df = pd.DataFrame(df, index=sorted(df.index, key=sort_key))\n",
    "            fname = f\"images/{d_type}_cm_{os}\"\n",
    "            if d_type == 'random_forest':\n",
    "                title = f\"Confusion Matrices for Inferred Output Data (Random Forest) on Phones Running {os} \\n by Calibration Settings\"\n",
    "            else:\n",
    "                title = f\"Confusion Matrices for Inferred Output Data (GIS) on Phones Running {os} \\n by Calibration Settings\"\n",
    "        else:\n",
    "            assert 0, f'INVALID d_type {d_type}'\n",
    "        df = df.reindex(\n",
    "            columns=['WALKING', 'BICYCLING', 'E_BIKE', 'ESCOOTER', 'CAR', 'BUS', 'SUBWAY', 'LIGHT_RAIL', 'TRAIN', 'NO_GT_START', 'NO_GT_MIDDLE', 'NO_GT_END']\n",
    "        ).fillna(0)\n",
    "#         df = df.div(df.sum(axis=1), axis=0)\n",
    "        cm = ax[k].imshow(df.transpose(), interpolation='nearest',  cmap=plt.cm.coolwarm, aspect='auto')\n",
    "        ax[k].set_title(role)\n",
    "    #     plt.colorbar(cm, ax=ax[0])\n",
    "        tick_marks = np.arange(len(df))\n",
    "        ax[k].set_yticks(np.arange(len(df.columns)))\n",
    "        ax[k].set_xticks(np.arange(len(df)))\n",
    "        ax[k].set_yticklabels(df)\n",
    "        ax[k].set_xticklabels(df.index, rotation=80)\n",
    "        color_thresh = df.max().max() / 4\n",
    "        for i, j in itertools.product(range(df.shape[1]), range(df.shape[0])  ):\n",
    "            ax[k].text(j, i, (int(df.transpose().iat[i,j])), horizontalalignment='center', \n",
    "                   color='white' \n",
    "                       if df.transpose().iat[i,j] < color_thresh \n",
    "#                            or df.transpose().iat[i,j] in df.max()\n",
    "                       else 'black')\n",
    "    fig.subplots_adjust(right=0.8)\n",
    "    cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])\n",
    "    fig.colorbar(cm, cax=cbar_ax)\n",
    "    plt.suptitle(title, weight='bold', size='x-large', y=y)\n",
    "\n",
    "    plt.savefig(fname=fname,  bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### plot the confusion matrices at each pipeline output stage on android/ios for select configuration settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_select_cm(os, role):\n",
    "    IIM = {0 : 'UNKNOWN', 1 : 'WALKING', 2 : 'BICYCLING', 3 : 'BUS', 4 : 'TRAIN', 5 : 'CAR', 6 : 'AIR_OR_HSR', 7 : 'SUBWAY', 8 : 'TRAM', 9 : 'LIGHT_RAIL'\n",
    "    }\n",
    "    CIM = {0 : 'IN_VEHICLE', 1 : 'BICYCLING', 2 : 'ON_FOOT', 3 : 'STILL', 4 : 'UNKNOWN', 5 : 'TILTING', 7 : 'WALKING', 8 : 'RUNNING', 9 : 'NONE', 10 : 'STOPPED_WHILE_IN_VEHICLE', 11 : 'AIR_OR_HSR'}\n",
    "    fig, ax = plt.subplots(1,5, figsize=(30,8), dpi=300, sharey=True)\n",
    "    y=.95\n",
    "    fig.text(0.5, -0.1, 'Predicted Label', ha='center', fontsize='xx-large')\n",
    "    fig.text(0.08, 0.5, 'True Label', va='center', rotation='vertical', fontsize='xx-large')\n",
    "    title = f\"Confusion Matrices for Phones Running {os}:{role}\"\n",
    "    fname = f\"images/selected_cm_{os}\"\n",
    "    for k, pv in enumerate(\n",
    "        [[pv_la, pv_sj, pv_ucb], \n",
    "         [mcv_la, mcv_sj, mcv_ucb],\n",
    "         [gcv_la, gcv_sj, gcv_ucb], \n",
    "         [rfv_la, rfv_sj, rfv_ucb], \n",
    "         [gisv_la, gisv_sj, gisv_ucb]]):\n",
    "        if k == 0:\n",
    "            df = pd.DataFrame(get_confusion_matrix(os, role, pv)).groupby('sensed_mode').sum()\n",
    "            df = pd.DataFrame(df, index=sorted(df.index, key=sort_key))\n",
    "        elif k == 1 or k == 2:\n",
    "            df = pd.DataFrame(get_confusion_matrix(os, role, pv)).groupby('sensed_mode').sum().rename(index=CIM)\n",
    "            df = pd.DataFrame(df, index=sorted(df.index, key=sort_key))\n",
    "\n",
    "        else:\n",
    "            df = pd.DataFrame(get_confusion_matrix(os, role, pv)).groupby('sensed_mode').sum().rename(index=IIM)\n",
    "            df = pd.DataFrame(df, index=sorted(df.index, key=sort_key))\n",
    "        df = df.reindex(\n",
    "            columns=['WALKING', 'BICYCLING', 'E_BIKE', 'ESCOOTER', 'CAR', 'BUS', 'SUBWAY', 'LIGHT_RAIL', 'TRAIN', 'NO_GT_START', 'NO_GT_MIDDLE', 'NO_GT_END']\n",
    "        ).fillna(0)\n",
    "        cm = ax[k].imshow(df.transpose(), interpolation='nearest',  cmap=plt.cm.coolwarm, aspect='auto')\n",
    "        title_map = {0 : f'raw output confusion matrix \\n{os}:{role}', \n",
    "                     1 : f'master clean output confusion matrix \\n{os}:{role}',\n",
    "                     2 : f'GIST clean output confusion matrix \\n{os}:{role}', \n",
    "                     3 : f'random forest output confusion matrix \\n{os}:{role}', \n",
    "                     4 : f'GIS output confusion matrix \\n{os}:{role}'}\n",
    "        ax[k].set_title(title_map[k])\n",
    "        tick_marks = np.arange(len(df))\n",
    "        ax[k].set_yticks(np.arange(len(df.columns)))\n",
    "        ax[k].set_xticks(np.arange(len(df)))\n",
    "        ax[k].set_yticklabels(df)\n",
    "        ax[k].set_xticklabels(df.index, rotation=80)\n",
    "        color_thresh = df.max().max() / 4\n",
    "        for i, j in itertools.product(range(df.shape[1]), range(df.shape[0])  ):\n",
    "            ax[k].text(j, i, (int(df.transpose().iat[i,j])), horizontalalignment='center', \n",
    "                   color='white' \n",
    "                       if df.transpose().iat[i,j] < color_thresh \n",
    "                       else 'black')\n",
    "    fig.subplots_adjust(right=0.8)\n",
    "    cbar_ax = fig.add_axes([0.825, 0.15, 0.025, 0.7])\n",
    "    fig.colorbar(cm, cax=cbar_ax)\n",
    "#     plt.suptitle(title, weight='bold', size='x-large', y=y)\n",
    "\n",
    "#     plt.savefig(fname=fname,  bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyzed Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### cleaned view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MASTER_DATA_LOC = \"bin/data/master_9b70c97\"\n",
    "master_spec = eisd.FileSpecDetails(MASTER_DATA_LOC, AUTHOR_EMAIL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GIS_DATA_LOC = \"bin/data/gis_9b679e3/\"\n",
    "gis_spec = eisd.FileSpecDetails(GIS_DATA_LOC, AUTHOR_EMAIL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mcv_la   = copy.deepcopy(eapv.create_analysed_view(pv_la, master_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\"))\n",
    "mcv_sj   = copy.deepcopy(eapv.create_analysed_view(pv_sj, master_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\"))\n",
    "mcv_ucb  = copy.deepcopy(eapv.create_analysed_view(pv_ucb, master_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gcv_la   = copy.deepcopy(eapv.create_analysed_view(pv_la, gis_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\"))\n",
    "gcv_sj   = copy.deepcopy(eapv.create_analysed_view(pv_sj, gis_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\"))\n",
    "gcv_ucb  = copy.deepcopy(eapv.create_analysed_view(pv_ucb, gis_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/cleaned_section\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### inferred view random forest\n",
    "\n",
    "sensed_db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfv_la   = copy.deepcopy(eapv.create_analysed_view(pv_la, master_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\"))\n",
    "rfv_sj   = copy.deepcopy(eapv.create_analysed_view(pv_sj, master_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\"))\n",
    "rfv_ucb  = copy.deepcopy(eapv.create_analysed_view(pv_ucb, master_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### inferred view GIS\n",
    "mobilitynet_db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gisv_la   = copy.deepcopy(eapv.create_analysed_view(pv_la, gis_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\"))\n",
    "gisv_sj   = copy.deepcopy(eapv.create_analysed_view(pv_sj, gis_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\"))\n",
    "gisv_ucb  = copy.deepcopy(eapv.create_analysed_view(pv_ucb, gis_spec, \"analysis/recreated_location\", \"analysis/cleaned_trip\", \"analysis/inferred_section\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Raw data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_cm('ios', [pv_la, pv_sj, pv_ucb], 'raw')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plot_cm('android', [pv_la, pv_sj, pv_ucb], 'raw')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cleaned data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plot_cm('ios', [mcv_la, mcv_sj, mcv_ucb], 'clean', CIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_cm('android', [mcv_la, mcv_sj, mcv_ucb], 'clean', CIM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Random Forrest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_cm('ios', [rfv_la,rfv_sj,rfv_ucb], 'random_forest', INDEX_MAP=IIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_cm('android', [rfv_la,rfv_sj,rfv_ucb], 'random_forest', INDEX_MAP=IIM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### GIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_cm('ios', [gisv_la,gisv_sj,gisv_ucb], 'gis', INDEX_MAP=IIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_cm('android', [gisv_la,gisv_sj,gisv_ucb], 'gis', INDEX_MAP=IIM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combined views"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_f_scores('ios')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_f_scores('android')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selected Setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_select_cm('ios', 'HAHFDC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_select_cm('android', 'HAMFDC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_f_scores_selected()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\n",
    "    \"GIS android \\t\",weighted_f_score('android', 'HAMFDC', [gisv_la,gisv_sj,gisv_ucb], GISBMM), '\\n',\n",
    "    \"GIS ios \\t\", weighted_f_score('ios', 'HAHFDC', [gisv_la,gisv_sj,gisv_ucb], GISBMM)\n",
    ")\n",
    "print(\n",
    "    \"Random Forest android \\t\",weighted_f_score('android', 'HAMFDC', [rfv_la,rfv_sj,rfv_ucb], GISBMM), '\\n',\n",
    "    \"Random Forest ios \\t\",weighted_f_score('ios', 'HAHFDC', [rfv_la,rfv_sj,rfv_ucb], GISBMM)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### get percentage of no sensed predicted mode for a given ground truth mode (in this case ground truth mode = walking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for pv_l in [[pv_la, pv_sj, pv_ucb],[mcv_la, mcv_sj, mcv_ucb],[rfv_la,rfv_sj,rfv_ucb],[gisv_la,gisv_sj,gisv_ucb]]:\n",
    "    df = pd.DataFrame(get_confusion_matrix('android', 'HAMFDC', pv_l)).groupby('sensed_mode').sum().rename(index=IIM)\n",
    "    w_n = (df['WALKING']['NO_SENSED_START'] + df['WALKING']['NO_SENSED_MIDDLE'] + df['WALKING']['NO_SENSED_END'])\n",
    "    print(\"NO_SENSED trip perdiction for WALKING android:HAMFDC: \\t\", w_n / df['WALKING'].sum())\n",
    "    df = pd.DataFrame(get_confusion_matrix('ios', 'HAHFDC', pv_l)).groupby('sensed_mode').sum().rename(index=IIM)\n",
    "    try:\n",
    "        w_n = (df['WALKING']['NO_SENSED_START'] + df['WALKING']['NO_SENSED_MIDDLE'] + df['WALKING']['NO_SENSED_END'])\n",
    "    except:\n",
    "        w_n = (df['WALKING']['NO_SENSED_START'] + df['WALKING']['NO_SENSED_MIDDLE'])\n",
    "    print(\"NO_SENSED trip perdiction for WALKING ios:HAHFDC: \\t\", w_n / df['WALKING'].sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unit Testing\n",
    "\n",
    "* get_binary_class_in_sec(os, role, pv, BASE_MODE, test=False, test_trip=None)\n",
    "* get_F_score(os, role, pv, BASE_MODE, beta=1, test=False, test_trip=None)\n",
    "* get_confusion_matrix(os, role, pv, test=False, test_trip=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example timelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### No sensed at the beggining, No GT at the end, Multimodal\n",
    "\n",
    "\n",
    "* pipeline timeline\n",
    "\n",
    "WALKING (0.5, 1.35) -> CYCLING (1.35, 2.7) -> WALKING (2.7, 3.5)\n",
    "\n",
    "* ground truth timeline\n",
    "\n",
    "WALKING (0, 1.05) -> BICYCLING (1.05, 3.1)\n",
    "\n",
    "#### binary classifiers\n",
    "    + TP \n",
    "        + [WALKING: 0.55] \n",
    "        + [CYCLING: 1.35]\n",
    "    + FP \n",
    "        + [WALKING: 1.10] \n",
    "        + [INVALID: 0.5]\n",
    "    + FN \n",
    "        + [WALKING: 0.50] \n",
    "        + [CYCLING: 0.70]\n",
    "        + [INVALID: 0.4]\n",
    "    + TN \n",
    "        + [WALKING: 1.35] \n",
    "        + [CYCLING: 1.45]\n",
    "        + [INVALID: 2.60]\n",
    "* $F_1$ score\n",
    "    + [WALKING: 0.41]\n",
    "    + [CYCLING: 0.79]\n",
    "    + [INVALID: ....]\n",
    "    \n",
    "#### Confusion Matrix\n",
    "\n",
    "\n",
    "sensed = [WALKING, CYCLING, NO_SENSED]\n",
    "\n",
    "ground truth = [WALKING, BICYCLING, NO_GT]\n",
    "\n",
    "origin top left\n",
    "\n",
    "    [0.55, 0.00, 0.50]\n",
    "    [0.70, 1.35, 0.00]\n",
    "    [0.40, 0.00, 0.00]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_trip = {\n",
    "    'ss_timeline'  : [{'start_ts' : 0.5, 'end_ts' : 1.35, 'mode' : 'WALKING'},\n",
    "                      {'start_ts' : 1.35, 'end_ts' : 2.7, 'mode' : 'CYCLING'}, \n",
    "                      {'start_ts' : 2.7, 'end_ts' : 3.5, 'mode' : 'WALKING'}],\n",
    "    'gts_timeline' : [{'start_ts' : 0, 'end_ts' : 1.05, 'mode' : 'WALKING'}, \n",
    "                      {'start_ts' : 1.05, 'end_ts' : 3.1, 'mode' : 'BICYCLING'}]\n",
    "}\n",
    "test_BMM = {'WALKING' : 'WALKING', 'CYCLING' : 'CYCLING', 'BICYCLING' : 'CYCLING', \n",
    "            'NO_SENSED' : 'INVALID', 'NO_GT' : 'INVALID', 'INVALID' : 'INVALID', \n",
    "            'NO_SENSED_START' : 'INVALID', 'NO_SENSED_MIDDLE' : 'INVALID', 'NO_SENSED_END' : 'INVALID', \n",
    "            'NO_GT_START' : 'INVALID', 'NO_GT_MIDDLE' : 'INVALID', 'NO_GT_END' : 'INVALID'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### get_binary_class_in_sec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_binary_class_in_sec(..., ..., ..., test_BMM, test=True, test_trip=test_trip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert round(res[0]['WALKING'], 2) == 0.55 and round(res[0]['CYCLING'], 2) == 1.35, f\"TP wrong\"\n",
    "assert round(res[1]['WALKING'], 2) == 1.10 and round(res[1]['INVALID'], 2) == 0.50 and len(res[1]) == 2, f\"FP wrong\"\n",
    "assert round(res[2]['WALKING'], 2) == 0.50 and round(res[2]['CYCLING'], 2) == 0.70 and round(res[2]['INVALID'], 2) == 0.40 and len(res[2]) == 3, f\"FN wrong\"\n",
    "assert round(res[3]['WALKING'], 2) == 1.35 and round(res[3]['CYCLING'], 2) == 1.45 and round(res[3]['INVALID'], 2) == 2.60 and len(res[2]) == 3, f\"TN wrong\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### get_F_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_F_score(..., ..., ..., test_BMM, beta=1, test=True, test_trip=test_trip)\n",
    "assert round(res['WALKING'],2) == 0.41 and round(res['CYCLING'], 2) == 0.79 and res['INVALID'] == 0, f\"F_1 scores wrong\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### get_confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_confusion_matrix(..., ..., ..., test=True, test_trip=test_trip)\n",
    "df = pd.DataFrame(res).groupby('sensed_mode').sum()\n",
    "assert 'WALKING' in df.index and 'CYCLING' in df.index and 'NO_SENSED_START' in df.index, f\"INCORRECT INDECIES \\n EXPECTED \\t 'WALKING', 'CYCLING', 'NO_SENSED' \\n GOT \\t \\t {df.index}\"\n",
    "assert df.loc['WALKING'].loc['WALKING'] == 0.55 and df.loc['WALKING'].loc['BICYCLING'] == 0.70 and round(df.loc['WALKING'].loc['NO_GT_END'],2) == 0.40\n",
    "assert df.loc['CYCLING'].loc['WALKING'] == 0 and df.loc['CYCLING'].loc['BICYCLING'] == 1.35 and round(df.loc['CYCLING'].loc['NO_GT_END'],2) == 0.0\n",
    "assert round(df.loc['NO_SENSED_START'].loc['WALKING'], 2) == 0.50 and df.loc['NO_SENSED_START'].loc['BICYCLING'] == 0.0 and round(df.loc['NO_SENSED_START'].loc['NO_GT_END'],2) == 0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### No sensed at beggining and end, multimodal\n",
    "\n",
    "* pipeline timeline\n",
    "WALKING (0.5, 1.5)\n",
    "\n",
    "* ground truth timeline\n",
    "WALKING (0, 1) -> (1, 2)\n",
    "\n",
    "#### Multiclass classifiers\n",
    "* TP\n",
    "    + WALKING [0.5]\n",
    "* FP\n",
    "    + WALKING [0.5]\n",
    "    + INVALID [1]\n",
    "* FN\n",
    "    + WALKING [0.5]\n",
    "    + CYCLING [1]\n",
    "* TN\n",
    "    + WALKING [0.5]\n",
    "    + CYCLING [1]\n",
    "    + INVALID [1]\n",
    "#### Confusion Matrix\n",
    "\n",
    "sensed mode = ['WALKING', 'NO_SENSED']\n",
    "\n",
    "ground truth = ['WALKING', 'CYCLING']\n",
    "\n",
    "    [0.5, 0.5]\n",
    "    [0.5, 0.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_trip = {\n",
    "    'ss_timeline'  : [{'start_ts' : 0.5, 'end_ts' : 1.5, 'mode' : 'WALKING'}],\n",
    "    'gts_timeline' : [{'start_ts' : 0, 'end_ts' : 1, 'mode' : 'WALKING'}, {'start_ts' : 1, 'end_ts' : 2, 'mode' : 'CYCLING'}]\n",
    "}\n",
    "test_BMM = {'WALKING' : 'WALKING', 'CYCLING' : \"CYCLING\", 'NO_SENSED' : 'INVALID', 'NO_GT' : 'INVALID', 'INVALID' : 'INVALID',\n",
    "            'NO_SENSED_START' : 'INVALID', 'NO_SENSED_MIDDLE' : 'INVALID', 'NO_SENSED_END' : 'INVALID', \n",
    "            'NO_GT_START' : 'INVALID', 'NO_GT_MIDDLE' : 'INVALID', 'NO_GT_END' : 'INVALID'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_binary_class_in_sec(..., ..., ..., test_BMM, test=True, test_trip=test_trip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert res[0]['WALKING'] == 0.5\n",
    "assert res[1]['INVALID'] == 1.0 and res[1]['WALKING'] == 0.5\n",
    "assert res[2]['WALKING'] == 0.5 and res[2]['CYCLING'] == 1 \n",
    "assert res[3]['WALKING'] == 0.5 and res[3]['CYCLING'] == 1 and res[3]['INVALID'] == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_confusion_matrix(..., ..., ..., test=True, test_trip=test_trip)\n",
    "df = pd.DataFrame(res).groupby('sensed_mode').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert df.loc['WALKING'].loc['WALKING'] == 0.5\n",
    "assert df.loc['WALKING'].loc['CYCLING'] == 0.5\n",
    "assert df.loc['NO_SENSED_START'].loc['WALKING'] == 0.5\n",
    "assert df.loc['NO_SENSED_END'].loc['CYCLING'] == 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### No ground truth at beggining and end, unimodal\n",
    "\n",
    "* pipeline timeline\n",
    "WALKING (0, 2)\n",
    "\n",
    "* ground truth timeline\n",
    "WALKING (0.5, 1.5)\n",
    "\n",
    "#### Multiclass classifiers\n",
    "* TP\n",
    "    + WALKING [1]\n",
    "* FP\n",
    "    + WALKING [1]\n",
    "* FN\n",
    "    + INVALID [1]\n",
    "* TN\n",
    "    + INVALID [1]\n",
    "* $F_1$ score\n",
    "    + ...\n",
    "#### Confusion Matrix\n",
    "\n",
    "sensed mode = ['WALKING']\n",
    "\n",
    "ground truth = ['WALKING', 'NO_GT]\n",
    "\n",
    "    [1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_trip = {\n",
    "    'gts_timeline'  : [{'start_ts' : 0.5, 'end_ts' : 1.5, 'mode' : 'WALKING'}],\n",
    "    'ss_timeline' : [{'start_ts' : 0, 'end_ts' : 2, 'mode' : 'WALKING'}]\n",
    "}\n",
    "test_BMM = {'WALKING' : 'WALKING', 'NO_SENSED' : 'INVALID', 'NO_GT' : 'INVALID', 'INVALID' : 'INVALID',\n",
    "            'NO_SENSED_START' : 'INVALID', 'NO_SENSED_MIDDLE' : 'INVALID', 'NO_SENSED_END' : 'INVALID', \n",
    "            'NO_GT_START' : 'INVALID', 'NO_GT_MIDDLE' : 'INVALID', 'NO_GT_END' : 'INVALID'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_binary_class_in_sec(..., ..., ..., test_BMM, test=True, test_trip=test_trip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert res[0]['WALKING'] == 1\n",
    "assert res[1]['WALKING'] == 1\n",
    "assert res[2]['INVALID'] == 1\n",
    "assert res[3]['INVALID'] == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_confusion_matrix(..., ..., ..., test=True, test_trip=test_trip)\n",
    "df = pd.DataFrame(res).groupby('sensed_mode').sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert df.loc['WALKING'].loc['WALKING'] == 1\n",
    "assert df.loc['WALKING'].loc['NO_GT_START'] == 0.5\n",
    "assert df.loc['WALKING'].loc['NO_GT_END'] == 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unimodal Sensed Timeline With Gap\n",
    "\n",
    "* sensed timeline\n",
    "    + WALKING(2, 4) -> CYCLING(6, 8)\n",
    "* ground truth timeline\n",
    "    + WALKING(0, 10)\n",
    "    \n",
    "#### Classifiers\n",
    "\n",
    "* TP\n",
    "    + WALKING [2]\n",
    "* FP\n",
    "    + NO_SENSED [6]\n",
    "    + CYCLING [2]\n",
    "* FN\n",
    "    + WALKING [8]\n",
    "* TN\n",
    "    + INVALID [4]\n",
    "    + CYCLING [8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_trip = [\n",
    "    {\n",
    "        'gts_timeline'  : [{'start_ts' : 0, 'end_ts' : 10, 'mode' : 'WALKING'}],\n",
    "        'ss_timeline' : [\n",
    "            {'start_ts' : 2, 'end_ts' : 4, 'mode' : 'WALKING'},\n",
    "            {'start_ts' : 6, 'end_ts' : 8, 'mode' : 'CYCLING'}\n",
    "        ]\n",
    "    }\n",
    "]\n",
    "test_BMM = {'WALKING' : 'WALKING', 'CYCLING' : 'CYCLING', 'NO_SENSED' : 'NO_SENSED', 'NO_GT' : 'NO_GT',\n",
    "            'NO_SENSED_START' : 'NO_SENSED', 'NO_SENSED_MIDDLE' : 'NO_SENSED', 'NO_SENSED_END' : 'NO_SENSED', \n",
    "            'NO_GT_START' : 'NO_GT', 'NO_GT_MIDDLE' : 'NO_GT', 'NO_GT_END' : 'NO_GT'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_binary_class_in_sec(..., ..., ..., test_BMM, test=True, test_trip=test_trip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert res[0]['WALKING'] == 2, f\"WALKING TP IS INCORRECT, SHOULD BE 4, GOT {res[0]['WALKING']}\"\n",
    "assert res[1]['NO_SENSED'] == 6, f\"NO_SENSED FP IS INCORRECT, SHOULD BE 6, GOT {res[1]['NO_SENSED']}\"\n",
    "assert res[1]['CYCLING'] == 2, f\"CYCLING FP IS INCORRECT, SHOULD BE 2, GOT {res[1]['CYCLING'] }\"\n",
    "assert res[2]['WALKING'] == 8, f\"WALKING FN IS INCORRECT, SHOULD BE 8, GOT {res[2]['WALKING'] }\"\n",
    "assert res[3]['NO_SENSED'] == 4, f\"NO_SENSED TN IS INCORRECT, SHOULD BE 4, GOT {res[3]['NO_SENSED']}\"\n",
    "assert res[3]['CYCLING'] == 8, f\"CYCLING TN IS INCORRECT, SHOULD BE 8, GOT {res[3]['CYCLING']}\"\n",
    "assert res[3]['NO_GT'] == 10, f\"NO_GT TN IS INCORRECT, SHOULD BE 10, GOT {res[3]['NO_GT']}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_confusion_matrix(..., ..., ..., test=True, test_trip=test_trip)\n",
    "df = pd.DataFrame(res).groupby('sensed_mode').sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unimodal Sensed Timeline With Gap\n",
    "### Flip of last trip\n",
    "* ground truth timeline\n",
    "    + WALKING(2, 4) -> CYCLING(6, 8)\n",
    "* sensed timeline\n",
    "    + WALKING(0, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flip = [{'gts_timeline' : test_trip[0]['ss_timeline'], 'ss_timeline' : test_trip[0]['gts_timeline']}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = get_binary_class_in_sec(..., ..., ..., test_BMM, test=True, test_trip=flip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert res[0]['WALKING'] == 2, f\"WALKING TP IS INCORRECT, SHOULD BE 4, GOT {res[0]['WALKING']}\"\n",
    "assert res[1]['WALKING'] == 8, f\"WALKING FP IS INCORRECT, SHOULD BE 8, GOT {res[2]['WALKING']}\"\n",
    "assert res[2]['NO_GT'] == 6, f\"NO_GT TN IS INCORRECT, SHOULD BE 6, GOT {res[3]['NO_GT'] }\"\n",
    "assert res[2]['CYCLING'] == 2, f\"CYCLING TN IS INCORRECT, SHOULD BE 2, GOT {res[3]['CYCLING'] }\"\n",
    "assert res[3]['CYCLING'] == 8, f\"CYCLING TN IS INCORRECT, SHOULD BE 8, GOT {res[3]['CYCLING']}\"\n",
    "assert res[3]['NO_SENSED'] == 10, f\"NO_SENSED TN IS INCORRECT, SHOULD BE 10, GOT {res[3]['NO_SENSED']}\"\n",
    "assert res[3]['NO_GT'] == 4, f\"NO_GT TN IS INCORRECT, SHOULD BE 4, GOT {res[3]['NO_GT']}\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## No ss, gts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_trip = [{'gts_timeline'  : [], 'ss_timeline' : []}]\n",
    "get_binary_class_in_sec(..., ..., ..., test_BMM, test=True, test_trip=test_trip)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## No ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_trip = [{'gts_timeline'  : [{'mode' : 'WALKING', 'start_ts' : 0, 'end_ts' : 1}], 'ss_timeline' : []}]\n",
    "get_binary_class_in_sec(..., ..., ..., test_BMM, test=True, test_trip=test_trip)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## No gts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_trip = [{'ss_timeline'  : [{'mode' : 'WALKING', 'start_ts' : 0, 'end_ts' : 1}], 'gts_timeline' : []}]\n",
    "get_binary_class_in_sec(..., ..., ..., test_BMM, test=True, test_trip=test_trip)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}