Integrate common base mode colors into public dashboard

viz_scripts/docker/environment36.dashboard.additions.yml

@@ -7,3 +7,4 @@ dependencies:
 - pip:
   - nbparameterise==0.6
   - devcron==0.4
+  - git+https://github.com/JGreenlee/e-mission-common@master

viz_scripts/energy_calculations.ipynb

@@ -106,7 +106,7 @@
    },
    "outputs": [],
    "source": [
-    "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n",
+    "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n",
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",

viz_scripts/generic_metrics.ipynb

@@ -90,7 +90,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)"
+    "colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose, values_to_translations_replaced = await scaffolding.mapping_color_labels(dynamic_labels)"
    ]
   },
   {
@@ -108,7 +108,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n",
+    "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n",
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
@@ -207,8 +207,8 @@
     "    # We will have text results corresponding to the axes for simplicity and consistency\n",
     "    text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n",
     "    \n",
-    "    plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n",
-    "                                    \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(expanded_ct, lambda df: (df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n",
+    "                                    \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n",
     "    plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: (df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False)), \n",
     "                                    \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n",
     "    \n",
@@ -258,8 +258,8 @@
     "    # Plot entries\n",
     "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)    \n",
     "    text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n",
-    "    plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
-    "                                    \"Labeled by user\\n (Confirmed trips)\", ax, text_results, colors_mode, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(expanded_ct_commute, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
+    "                                    \"Labeled by user\\n (Confirmed trips)\", ax, text_results, colors_mode, debug_df, values_to_translations)\n",
     "    set_title_and_save(fig, text_results, plot_title, file_name)\n",
     "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n",
     "    plt.clf()\n",
@@ -291,8 +291,8 @@
     "try:\n",
     "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n",
     "    text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n",
-    "    plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"Trip_purpose\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
-    "                                    \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax, text_results, colors_purpose, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"merge_purpose_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
+    "                                    \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax, text_results, colors_purpose, debug_df, value_to_translations_purpose)\n",
     "    set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n",
     "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n",
     "    plt.clf()\n",
@@ -335,16 +335,16 @@
     "\n",
     "    ## We do an existence check for the labeled df because we want to display the sensed value even if we don't have the labeled value\n",
     "    ## but we don't need to have an existence check for sensed because in that case we will have no data to display\n",
-    "    expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"Mode_confirm\" in expanded_ct.columns else None\n",
+    "    expanded_ct_u80 = expanded_ct.loc[(expanded_ct['distance'] <= cutoff)] if \"merge_mode_confirm\" in expanded_ct.columns else None\n",
     "    expanded_ct_sensed_u80 = expanded_ct_sensed.loc[(expanded_ct_sensed['distance'] <= cutoff)]\n",
     "    sensed_u80_quality_text = f\"{len(expanded_ct_sensed_u80)} trips ({round(len(expanded_ct_sensed_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_sensed_u80)} {sensed_match.group(3)}\"\n",
     "    labeled_u80_quality_text = f\"{len(expanded_ct_u80)} trips ({round(len(expanded_ct_u80)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(expanded_ct_u80)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(expanded_ct_u80)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n",
     "    \n",
     "    # Plot entries\n",
     "    fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n",
     "    text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n",
-    "    plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
-    "                                    \"Labeled by user\\n\"+labeled_u80_quality_text, ax[0], text_results[0], colors_mode, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(expanded_ct_u80, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
+    "                                    \"Labeled by user\\n\"+labeled_u80_quality_text, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n",
     "    plot_and_text_stacked_bar_chart(expanded_ct_sensed_u80, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
     "                                    \"Sensed by OpenPATH\\n\"+sensed_u80_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n",
     "    set_title_and_save(fig, text_results, plot_title_no_quality, file_name)\n",
@@ -383,8 +383,8 @@
     "    fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n",
     "    \n",
     "    text_results = [[\"Unmodified Alt Text\", \"Unmodified HTML\"], [\"Unmodified Alt Text\", \"Unmodified HTML\"]]\n",
-    "    plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
-    "                                    \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(expanded_ct, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
+    "                                    \"Labeled by user\\n\"+stacked_bar_quality_text_labeled, ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n",
     "    plot_and_text_stacked_bar_chart(expanded_ct_sensed, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
     "                                    \"Sensed by OpenPATH\\n\"+stacked_bar_quality_text_sensed, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n",
     "    set_title_and_save(fig, text_results, plot_title_no_quality, file_name)    \n",
@@ -419,15 +419,15 @@
     "try:\n",
     "    ## We do an existence check for the labeled df because we want to display the sensed value even if we don't have the labeled value\n",
     "    ## but we don't need to have an existence check for sensed because in that case we will have no data to display\n",
-    "    labeled_land_trips_df = expanded_ct[expanded_ct['Mode_confirm'] != \"Airplane\"] if \"Mode_confirm\" in expanded_ct.columns else None\n",
+    "    labeled_land_trips_df = expanded_ct[expanded_ct['merge_mode_confirm'] != \"air\"] if \"merge_mode_confirm\" in expanded_ct.columns else None\n",
     "    sensed_land_trips_df = expanded_ct_sensed[expanded_ct_sensed['primary_mode'] != \"AIR_OR_HSR\"]\n",
     "    \n",
     "    sensed_land_quality_text = f\"{len(sensed_land_trips_df)} trips ({round(len(sensed_land_trips_df)/len(expanded_ct_sensed)*100)}% of all trips)\\nfrom {scaffolding.unique_users(sensed_land_trips_df)} {sensed_match.group(3)}\"\n",
-    "    labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"Mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n",
+    "    labeled_land_quality_text = f\"{len(labeled_land_trips_df)} trips ({round(len(labeled_land_trips_df)/len(expanded_ct)*100)}% of all labeled,\\n{round(len(labeled_land_trips_df)/len(expanded_ct_sensed)*100)}%) of all trips)\\nfrom {scaffolding.unique_users(labeled_land_trips_df)} {sensed_match.group(3)}\" if \"merge_mode_confirm\" in expanded_ct.columns else \"0 labeled trips\"\n",
     "\n",
     "    fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15,2*2), sharex=True)\n",
-    "    plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"Mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
-    "                                    \"Labeled by user\\n\"+labeled_land_quality_text,  ax[0], text_results[0], colors_mode, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(labeled_land_trips_df, lambda df: df.groupby(\"merge_mode_confirm\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
+    "                                    \"Labeled by user\\n\"+labeled_land_quality_text,  ax[0], text_results[0], colors_mode, debug_df, values_to_translations)\n",
     "    plot_and_text_stacked_bar_chart(sensed_land_trips_df, lambda df: df.groupby(\"primary_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
     "                                    \"Sensed by OpenPATH\\n\"+sensed_land_quality_text, ax[1], text_results[1], colors_sensed, debug_df_sensed)\n",
     "    set_title_and_save(fig, text_results, plot_title_no_quality, file_name)    \n",

viz_scripts/generic_metrics_sensed.ipynb

@@ -96,7 +96,7 @@
     "        expanded_ct[\"primary_mode\"] = expanded_ct.ble_sensed_summary.apply(lambda md: max(md[\"distance\"], key=md[\"distance\"].get))\n",
     "        unique_keys = expanded_ct.groupby(\"primary_mode\").agg({distance_col: \"count\"}).index\n",
     "        print(unique_keys)\n",
-    "        colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels({},  dict(zip(unique_keys, unique_keys)), {})\n",
+    "        colors_mode, colors_purpose, colors_sensed = await scaffolding.mapping_color_labels({},  dict(zip(unique_keys, unique_keys)), {})\n",
     "        colors_sensed = colors_mode\n",
     "except ValueError as e:\n",
     "    print(\"Got ValueError \", e)"

viz_scripts/generic_timeseries.ipynb

@@ -87,7 +87,7 @@
    },
    "outputs": [],
    "source": [
-    "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n",
+    "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n",
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",

viz_scripts/mode_specific_metrics.ipynb

@@ -103,7 +103,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "colors_mode, colors_purpose, colors_sensed = scaffolding.mapping_color_labels(dynamic_labels, dic_re, dic_pur)"
+    "colors_mode, colors_replaced, colors_purpose, colors_sensed, values_to_translations, value_to_translations_purpose, value_to_translations_replaced = await scaffolding.mapping_color_labels(dynamic_labels)"
    ]
   },
   {
@@ -121,7 +121,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n",
+    "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n",
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",
@@ -193,8 +193,8 @@
     "try:\n",
     "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n",
     "    text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n",
-    "    plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"Trip_purpose\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False),\n",
-    "                                    f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_purpose, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_purpose_confirm\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False),\n",
+    "                                    f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_purpose, debug_df, value_to_translations_purpose)\n",
     "    plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n",
     "    set_title_and_save(fig, text_results, plot_title, file_name)\n",
     "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n",
@@ -228,8 +228,8 @@
     "try:\n",
     "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n",
     "    text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n",
-    "    plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"Replaced_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
-    "                                    \"Labeled by user\\n (Trip distance)\", ax, text_results, colors_mode, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_replaced_mode\").agg({distance_col: 'sum'}).sort_values(by=distance_col, ascending=False), \n",
+    "                                    \"Labeled by user\\n (Trip distance)\", ax, text_results, colors_replaced, debug_df, value_to_translations_replaced)\n",
     "    plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n",
     "    set_title_and_save(fig, text_results, plot_title, file_name)\n",
     "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n",
@@ -263,8 +263,8 @@
     "try:\n",
     "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15,2*1), sharex=True)\n",
     "    text_results = [\"Unmodified Alt Text\", \"Unmodified HTML\"]\n",
-    "    plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"Replaced_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
-    "                                    f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_mode, debug_df)\n",
+    "    plot_and_text_stacked_bar_chart(data_eb, lambda df: df.groupby(\"merge_replaced_mode\").agg({distance_col: 'count'}).sort_values(by=distance_col, ascending=False), \n",
+    "                                    f\"Labeled `{mode_of_interest}` by user\", ax, text_results, colors_replaced, debug_df, value_to_translations_replaced)\n",
     "    plot_title = plot_title_no_quality + \"\\n\" + f\"For {mode_of_interest}: \" + quality_text\n",
     "    set_title_and_save(fig, text_results, plot_title, file_name)\n",
     "except (AttributeError, KeyError, pd.errors.UndefinedVariableError) as e:\n",

viz_scripts/mode_specific_timeseries.ipynb

@@ -96,7 +96,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "expanded_ct, file_suffix, quality_text, debug_df = scaffolding.load_viz_notebook_data(year,\n",
+    "expanded_ct, file_suffix, quality_text, debug_df = await scaffolding.load_viz_notebook_data(year,\n",
     "                                                                            month,\n",
     "                                                                            program,\n",
     "                                                                            study_type,\n",