From 7050320b93d07427ec93dbbda192086099197e1d Mon Sep 17 00:00:00 2001 From: Robin Holzinger Date: Fri, 27 Sep 2024 12:12:06 +0200 Subject: [PATCH] heatmap --- analytics/plotting/common/heatmap.py | 58 +++- analytics/plotting/rh_thesis/TODO.md | 16 +- .../drift/arxiv_heatmap_single.ipynb | 286 ++++++++++++++++++ .../rh_thesis/drift/hp_heatmap_single.ipynb | 286 ++++++++++++++++++ .../drift/yb_cost_perf_tradeoff.ipynb | 30 +- .../triggers_simple/arxiv_cost.ipynb | 37 +++ .../arxiv_cost_perf_tradeoff.ipynb | 14 +- .../hp_cost_perf_tradeoff.ipynb | 18 +- .../yb_cost_perf_tradeoff.ipynb | 19 +- 9 files changed, 738 insertions(+), 26 deletions(-) create mode 100644 analytics/plotting/rh_thesis/drift/arxiv_heatmap_single.ipynb create mode 100644 analytics/plotting/rh_thesis/drift/hp_heatmap_single.ipynb diff --git a/analytics/plotting/common/heatmap.py b/analytics/plotting/common/heatmap.py index 5cf1faa2f..5014b3a39 100644 --- a/analytics/plotting/common/heatmap.py +++ b/analytics/plotting/common/heatmap.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Literal import matplotlib.patches as patches import pandas as pd @@ -14,6 +14,31 @@ from analytics.plotting.common.font import setup_font +def get_fractional_index(dates: pd.Series, query_date: pd.Timestamp, fractional: bool = True) -> float: + """Given a list of Period objects (dates) and a query_date as a Period, + return the interpolated fractional index between two period indices if the + query_date lies between them.""" + # Ensure query_date is within the bounds of the period range + if query_date < dates[0].start_time: + return -1 # -1 before first index + + if query_date > dates[-1].start_time: + return len(dates) # +1 after last index + + # Find the two periods where the query_date falls in between + for i in range(len(dates) - 1): + if dates[i].start_time <= query_date <= dates[i + 1].start_time: + # Perform linear interpolation, assuming equal length periods + return i + ( + ((query_date - dates[i].start_time) / (dates[i + 1].start_time - dates[i].start_time)) + if fractional + else 0 + ) + + # If query_date is exactly one of the dates + return dates.get_loc(query_date) + + def build_heatmap( heatmap_data: pd.DataFrame, y_ticks: list[int] | list[str] | None = None, @@ -39,7 +64,8 @@ def build_heatmap( grid_alpha: float = 0.0, disable_horizontal_grid: bool = False, df_logs_models: pd.DataFrame | None = None, - triggers: dict[int, list[pd.Timestamp]] = {}, + triggers: dict[int, pd.DataFrame] = {}, + x_axis: Literal["int", "period"] = "year", ) -> Figure | Axes: init_plot() setup_font(small_label=True, small_title=True) @@ -92,7 +118,7 @@ def build_heatmap( ax.set_xlabel(x_label) if not x_ticks and not x_custom_ticks: ax.set_xticks( - ticks=[x + 0.5 for x in range(0, 2010 - 1930 + 1, 20)], + ticks=[x + 0.5 for x in range(0, 2010 - 1930 + 1, 20)], # TODO: check 0.5 labels=[x for x in range(1930, 2010 + 1, 20)], rotation=0, # ha='right' @@ -184,11 +210,31 @@ def build_heatmap( if df_logs_models is not None: for type_, dashed in [("train", False), ("usage", False), ("train", True)]: for active_ in df_logs_models.iterrows(): - x_start = active_[1][f"{type_}_start"].year - 1930 - x_end = active_[1][f"{type_}_end"].year - 1930 + if x_axis == "year": + x_start = active_[1][f"{type_}_start"].year - 1930 + x_end = active_[1][f"{type_}_end"].year - 1930 + else: + # start_idx = get_fractional_index(heatmap_data.columns, start_date) + # end_idx = get_fractional_index(heatmap_data.columns, end_date) + # x_start = heatmap_data.columns.get_loc(active_[1][f"{type_}_start"]) + # x_end = heatmap_data.columns.get_loc(active_[1][f"{type_}_end"]) + x_start = get_fractional_index( + heatmap_data.columns, + active_[1][f"{type_}_start"], + fractional=False, + ) + x_end = get_fractional_index( + heatmap_data.columns, + active_[1][f"{type_}_end"], + fractional=False, + ) + y = active_[1]["model_idx"] rect = plt.Rectangle( - (x_start, y - 1), # y: 0 based index, model_idx: 1 based index + ( + x_start, + y - 1, + ), # y: 0 based index, model_idx: 1 based index x_end - x_start, 1, edgecolor="White" if type_ == "train" else "Black", diff --git a/analytics/plotting/rh_thesis/TODO.md b/analytics/plotting/rh_thesis/TODO.md index 0faef245d..27e973a65 100644 --- a/analytics/plotting/rh_thesis/TODO.md +++ b/analytics/plotting/rh_thesis/TODO.md @@ -1,3 +1,17 @@ drift: -- plot arxiv / huffpost +- plot arxiv + +performance: + +- 1 cost plot +- 1 single pipeline heatmap +- 1 multi pipeline heatmap for every dataset (including best of every subtype) + +cost: + +- 1 dummy plot + +discussion: + +- tradeoff plot: 1 per dataset diff --git a/analytics/plotting/rh_thesis/drift/arxiv_heatmap_single.ipynb b/analytics/plotting/rh_thesis/drift/arxiv_heatmap_single.ipynb new file mode 100644 index 000000000..100651d98 --- /dev/null +++ b/analytics/plotting/rh_thesis/drift/arxiv_heatmap_single.ipynb @@ -0,0 +1,286 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import pandas as pd\n", + "\n", + "from analytics.app.data.load import list_pipelines\n", + "from analytics.app.data.transform import dfs_models_and_evals, logs_dataframe\n", + "\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipelines_dir = Path(\n", + " \"/Users/robinholzinger/robin/dev/eth/modyn-robinholzi-data/data/triggering/arxiv/21_datadrift_dynamic\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipelines = list_pipelines(pipelines_dir)\n", + "max_pipeline_id = max(pipelines.keys())\n", + "pipelines" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from analytics.app.data.load import load_pipeline_logs\n", + "\n", + "pipeline_logs = {p_id: load_pipeline_logs(p_id, pipelines_dir) for (p_id, (_, p_path)) in pipelines.items()}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# mode:\n", + "pipeline_id = 771 # hp drifttrigger_mmd-rollavg-2.0-20_int1500_win1y\n", + "\n", + "# doesn't do anything unless include_composite_model = True\n", + "composite_model_variant = \"currently_active_model\"\n", + "\n", + "patch_yearbook = True\n", + "dataset_id = \"huffpost_kaggle_test\"\n", + "eval_handler = \"periodic-current\"\n", + "metric = \"Accuracy\"\n", + "include_composite_model = False" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Wrangle data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipeline_log = pipeline_logs[pipeline_id]\n", + "pipeline_ref = f\"{pipeline_id}\".zfill(len(str(max_pipeline_id))) + f\" - {pipelines[pipeline_id][0]}\"\n", + "\n", + "df_all = logs_dataframe(pipeline_log, pipeline_ref)\n", + "\n", + "df_logs_models, _, df_eval_single = dfs_models_and_evals(\n", + " # subtracting would interfere with yearbook patching\n", + " pipeline_log,\n", + " df_all[\"sample_time\"].max(),\n", + " pipeline_ref,\n", + ")\n", + "\n", + "df_adjusted = df_eval_single\n", + "\n", + "\n", + "df_adjusted = df_adjusted[\n", + " (df_adjusted[\"dataset_id\"] == dataset_id)\n", + " & (df_adjusted[\"eval_handler\"] == eval_handler)\n", + " & (df_adjusted[\"metric\"] == metric)\n", + "]\n", + "\n", + "# in percent (0-100)\n", + "df_adjusted[\"value\"] = df_adjusted[\"value\"] * 100" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_adjusted = df_adjusted.sort_values(by=[\"interval_center\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Add composite model\n", + "\n", + "assert df_adjusted[\"pipeline_ref\"].nunique() <= 1\n", + "# add the pipeline time series which is the performance of different models stitched together dep.\n", + "# w.r.t which model was active\n", + "pipeline_composite_model = df_adjusted[df_adjusted[composite_model_variant]]\n", + "pipeline_composite_model[\"model_idx\"] = 0\n", + "pipeline_composite_model[\"id_model\"] = 0\n", + "\n", + "label_map = {k: f\"{k}\" for k, v in df_adjusted[[\"model_idx\", \"id_model\"]].values}\n", + "label_map[0] = \"Pipeline composite model\"\n", + "\n", + "if include_composite_model:\n", + " df_adjusted = pd.concat([pipeline_composite_model, df_adjusted])\n", + "else:\n", + " df_adjusted[\"model_idx\"] = df_adjusted[\"model_idx\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create Plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_adjusted = df_adjusted.sort_values(by=[\"interval_center\"])\n", + "df_adjusted[\"interval_center\"] = df_adjusted[\"interval_center\"].dt.to_period(\"M\")\n", + "df_adjusted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_train_end_years_per_model = df_logs_models[[\"model_idx\", \"real_train_end\"]]\n", + "df_train_end_years_per_model[\"real_train_end\"] = df_train_end_years_per_model[\"real_train_end\"].dt.to_period(\"M\")\n", + "df_train_end_years_per_model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_merged = df_adjusted.merge(df_train_end_years_per_model, on=\"model_idx\", how=\"left\")\n", + "df_merged" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_merged.groupby([\"real_train_end\", \"interval_center\"]).size()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# build heatmap matrix dataframe:\n", + "df_merged[\"real_train_end\"] = df_merged[\"real_train_end\"].apply(lambda x: pd.Period(x, freq=\"M\"))\n", + "heatmap_data = df_merged.pivot(index=[\"real_train_end\"], columns=\"interval_center\", values=\"value\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "heatmap_data.index.min(), heatmap_data.index.max()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "heatmap_data.index" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from analytics.plotting.common.heatmap import build_heatmap\n", + "from analytics.plotting.common.save import save_plot\n", + "\n", + "fig = build_heatmap(\n", + " heatmap_data,\n", + " reverse_col=True,\n", + " x_custom_ticks=[\n", + " (i, f\"{period.to_timestamp().strftime('%b %Y')}\".replace(\" \", \"\\n\"))\n", + " for i, period in list(enumerate(heatmap_data.columns))[::1]\n", + " if period in [pd.Period(\"Apr 2014\"), pd.Period(\"Jul 2018\"), pd.Period(\"Jan 2022\")]\n", + " ],\n", + " y_custom_ticks=[\n", + " (i + 0.5, f\"{period.to_timestamp().strftime('%b %Y')}\")\n", + " for i, period in list(enumerate(heatmap_data.index))[::1]\n", + " ],\n", + " y_label=\"Trained up to\",\n", + " x_label=\"Evaluation Year\",\n", + " title_label=\"HuffPost Dynamic Threshold\\nRolling Average: Δ +200%\",\n", + " color_label=\"Accuracy %\",\n", + " width_factor=0.6,\n", + " height_factor=0.61,\n", + " # grid_alpha=0.4,\n", + " grid_alpha=0.0,\n", + " # disable_horizontal_grid=True,\n", + " # cbar=False,\n", + " df_logs_models=df_logs_models,\n", + " x_axis=\"period\",\n", + ")\n", + "save_plot(fig, \"arxiv_trigger_heatmap_drift_single_dynamic\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/analytics/plotting/rh_thesis/drift/hp_heatmap_single.ipynb b/analytics/plotting/rh_thesis/drift/hp_heatmap_single.ipynb new file mode 100644 index 000000000..26784853b --- /dev/null +++ b/analytics/plotting/rh_thesis/drift/hp_heatmap_single.ipynb @@ -0,0 +1,286 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import pandas as pd\n", + "\n", + "from analytics.app.data.load import list_pipelines\n", + "from analytics.app.data.transform import dfs_models_and_evals, logs_dataframe\n", + "\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipelines_dir = Path(\n", + " \"/Users/robinholzinger/robin/dev/eth/modyn-robinholzi-data/data/triggering/huffpost/21_datadrift_dynamic\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipelines = list_pipelines(pipelines_dir)\n", + "max_pipeline_id = max(pipelines.keys())\n", + "pipelines" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from analytics.app.data.load import load_pipeline_logs\n", + "\n", + "pipeline_logs = {p_id: load_pipeline_logs(p_id, pipelines_dir) for (p_id, (_, p_path)) in pipelines.items()}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# mode:\n", + "pipeline_id = 771 # hp drifttrigger_mmd-rollavg-2.0-20_int1500_win1y\n", + "\n", + "# doesn't do anything unless include_composite_model = True\n", + "composite_model_variant = \"currently_active_model\"\n", + "\n", + "patch_yearbook = True\n", + "dataset_id = \"huffpost_kaggle_test\"\n", + "eval_handler = \"periodic-current\"\n", + "metric = \"Accuracy\"\n", + "include_composite_model = False" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Wrangle data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipeline_log = pipeline_logs[pipeline_id]\n", + "pipeline_ref = f\"{pipeline_id}\".zfill(len(str(max_pipeline_id))) + f\" - {pipelines[pipeline_id][0]}\"\n", + "\n", + "df_all = logs_dataframe(pipeline_log, pipeline_ref)\n", + "\n", + "df_logs_models, _, df_eval_single = dfs_models_and_evals(\n", + " # subtracting would interfere with yearbook patching\n", + " pipeline_log,\n", + " df_all[\"sample_time\"].max(),\n", + " pipeline_ref,\n", + ")\n", + "\n", + "df_adjusted = df_eval_single\n", + "\n", + "\n", + "df_adjusted = df_adjusted[\n", + " (df_adjusted[\"dataset_id\"] == dataset_id)\n", + " & (df_adjusted[\"eval_handler\"] == eval_handler)\n", + " & (df_adjusted[\"metric\"] == metric)\n", + "]\n", + "\n", + "# in percent (0-100)\n", + "df_adjusted[\"value\"] = df_adjusted[\"value\"] * 100" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_adjusted = df_adjusted.sort_values(by=[\"interval_center\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Add composite model\n", + "\n", + "assert df_adjusted[\"pipeline_ref\"].nunique() <= 1\n", + "# add the pipeline time series which is the performance of different models stitched together dep.\n", + "# w.r.t which model was active\n", + "pipeline_composite_model = df_adjusted[df_adjusted[composite_model_variant]]\n", + "pipeline_composite_model[\"model_idx\"] = 0\n", + "pipeline_composite_model[\"id_model\"] = 0\n", + "\n", + "label_map = {k: f\"{k}\" for k, v in df_adjusted[[\"model_idx\", \"id_model\"]].values}\n", + "label_map[0] = \"Pipeline composite model\"\n", + "\n", + "if include_composite_model:\n", + " df_adjusted = pd.concat([pipeline_composite_model, df_adjusted])\n", + "else:\n", + " df_adjusted[\"model_idx\"] = df_adjusted[\"model_idx\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create Plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_adjusted = df_adjusted.sort_values(by=[\"interval_center\"])\n", + "df_adjusted[\"interval_center\"] = df_adjusted[\"interval_center\"].dt.to_period(\"M\")\n", + "df_adjusted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_train_end_years_per_model = df_logs_models[[\"model_idx\", \"real_train_end\"]]\n", + "df_train_end_years_per_model[\"real_train_end\"] = df_train_end_years_per_model[\"real_train_end\"].dt.to_period(\"M\")\n", + "df_train_end_years_per_model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_merged = df_adjusted.merge(df_train_end_years_per_model, on=\"model_idx\", how=\"left\")\n", + "df_merged" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_merged.groupby([\"real_train_end\", \"interval_center\"]).size()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# build heatmap matrix dataframe:\n", + "df_merged[\"real_train_end\"] = df_merged[\"real_train_end\"].apply(lambda x: pd.Period(x, freq=\"M\"))\n", + "heatmap_data = df_merged.pivot(index=[\"real_train_end\"], columns=\"interval_center\", values=\"value\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "heatmap_data.index.min(), heatmap_data.index.max()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "heatmap_data.index" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from analytics.plotting.common.heatmap import build_heatmap\n", + "from analytics.plotting.common.save import save_plot\n", + "\n", + "fig = build_heatmap(\n", + " heatmap_data,\n", + " reverse_col=True,\n", + " x_custom_ticks=[\n", + " (i, f\"{period.to_timestamp().strftime('%b %Y')}\".replace(\" \", \"\\n\"))\n", + " for i, period in list(enumerate(heatmap_data.columns))[::1]\n", + " if period in [pd.Period(\"Apr 2014\"), pd.Period(\"Jul 2018\"), pd.Period(\"Jan 2022\")]\n", + " ],\n", + " y_custom_ticks=[\n", + " (i + 0.5, f\"{period.to_timestamp().strftime('%b %Y')}\")\n", + " for i, period in list(enumerate(heatmap_data.index))[::1]\n", + " ],\n", + " y_label=\"Trained up to\",\n", + " x_label=\"Evaluation Year\",\n", + " title_label=\"HuffPost Dynamic Threshold\\nRolling Average: Δ +200%\",\n", + " color_label=\"Accuracy %\",\n", + " width_factor=0.6,\n", + " height_factor=0.61,\n", + " # grid_alpha=0.4,\n", + " grid_alpha=0.0,\n", + " # disable_horizontal_grid=True,\n", + " # cbar=False,\n", + " df_logs_models=df_logs_models,\n", + " x_axis=\"period\",\n", + ")\n", + "save_plot(fig, \"hp_trigger_heatmap_drift_single_dynamic\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/analytics/plotting/rh_thesis/drift/yb_cost_perf_tradeoff.ipynb b/analytics/plotting/rh_thesis/drift/yb_cost_perf_tradeoff.ipynb index fa419159c..379988a5f 100644 --- a/analytics/plotting/rh_thesis/drift/yb_cost_perf_tradeoff.ipynb +++ b/analytics/plotting/rh_thesis/drift/yb_cost_perf_tradeoff.ipynb @@ -49,6 +49,23 @@ " assert dir.exists()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from modyn.supervisor.internal.pipeline_executor.models import StageLog\n", + "\n", + "df = StageLog.df(\n", + " [x for x in pipeline_logs.get(63).supervisor_logs.stage_runs if x.id == PipelineStage.TRAIN.name], extended=True\n", + ")\n", + "\n", + "max_trigger_idx = df[\"trigger_idx\"].idxmax()\n", + "time_at_trainer = df.loc[max_trigger_idx, \"train_time_at_trainer\"]\n", + "time_at_trainer" + ] + }, { "cell_type": "code", "execution_count": null, @@ -329,9 +346,9 @@ " )\n", "].copy()\n", "renamed[\"Trigger Type\"] = renamed[\"pipeline_ref\"].apply(\n", - " lambda x: \"Dyn. Quantile [0.05/0.1/0.15]\"\n", + " lambda x: \"Dyn. Quantile % [0.05/0.1/0.15]\"\n", " if \"quant\" in x\n", - " else (\"Time [2.0/1.0/0.05/0.5]\" if \"roll\" in x else (\"static MMD threshold\\n[0.03/0.05/0.07/0.09/0.12]\"))\n", + " else (\"Roll. Avg Δ [2.0/1.0/0.05/0.5]\" if \"roll\" in x else (\"static MMD threshold\\n[0.03/0.05/0.07/0.09/0.12]\"))\n", ")\n", "renamed" ] @@ -375,7 +392,7 @@ " x_label=\"Total Cost (Minutes)\",\n", " y_label=\"Mean Accuracy %\",\n", " height_factor=0.6,\n", - " width_factor=0.65,\n", + " width_factor=0.7,\n", ")\n", "\n", "save_plot(fig, \"tradeoff_drift_yearbook_cost_performance\")" @@ -401,13 +418,6 @@ "\n", "save_plot(fig, \"tradeoff_drift_yearbook_triggers_cost\")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/analytics/plotting/rh_thesis/triggers_simple/arxiv_cost.ipynb b/analytics/plotting/rh_thesis/triggers_simple/arxiv_cost.ipynb index 85910958b..1a25c6d8c 100644 --- a/analytics/plotting/rh_thesis/triggers_simple/arxiv_cost.ipynb +++ b/analytics/plotting/rh_thesis/triggers_simple/arxiv_cost.ipynb @@ -220,6 +220,43 @@ "save_plot(fig, \"arxiv_amount-trigger-cost-matrix\")\n", "# not interesting: note that for 250 samples we see multiple trigger at the same timestamp" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Plot 100k amount and 2y time trigger together" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig = plot_cost_matrix(\n", + " df_new,\n", + " [269, 271],\n", + " grid_alpha=0.75,\n", + " title_map={\n", + " 269: \"TimeTrigger 2 years\",\n", + " 271: \"AmountTrigger 100k samples\",\n", + " },\n", + " height_factor=1.2,\n", + " width_factor=1.0,\n", + " duration_ylabel=\"Duration (min)\",\n", + " cumulative_ylabel=\"Cumulative Duration (min)\",\n", + " x_date_locator=FixedLocator([mdates.date2num(pd.Timestamp(d)) for d in [\"2000-01-01\", \"2009-01-01\", \"2018-01-01\"]]),\n", + " x_date_formatter=FixedFormatter([str(year) for year in [\"Jan 2000\", \"Jan 2009\", \"Jan 2018\"]]),\n", + " x_lim=(pd.Timestamp(\"1995-01-01\"), pd.Timestamp(\"2024-09-01\")),\n", + " y_ticks_cumulative=[x for x in range(0, 1000, 200)],\n", + " y_lim_cumulative=(0, 1000),\n", + " y_minutes=True,\n", + " y_minutes_cumulative=True,\n", + ")\n", + "\n", + "save_plot(fig, \"arxiv_timeamount-trigger-cost-matrix\")" + ] } ], "metadata": { diff --git a/analytics/plotting/rh_thesis/triggers_simple/arxiv_cost_perf_tradeoff.ipynb b/analytics/plotting/rh_thesis/triggers_simple/arxiv_cost_perf_tradeoff.ipynb index b98fe8441..107c73b77 100644 --- a/analytics/plotting/rh_thesis/triggers_simple/arxiv_cost_perf_tradeoff.ipynb +++ b/analytics/plotting/rh_thesis/triggers_simple/arxiv_cost_perf_tradeoff.ipynb @@ -325,7 +325,10 @@ "source": [ "# extract number if samples from pipeline name\n", "amount_policies = \"/\".join(\n", - " [str(x) for x in [str(y).replace(\" samples\", \"\") for y in merged[merged[\"type\"] == \"amount\"][\"pipeline_ref\"]]]\n", + " [\n", + " f\"{int(x) // 1000}k\"\n", + " for x in [str(y).replace(\" samples\", \"\") for y in merged[merged[\"type\"] == \"amount\"][\"pipeline_ref\"]]\n", + " ]\n", ")\n", "time_policies = \"/\".join(\n", " [\n", @@ -341,7 +344,7 @@ "\n", "renamed = merged.copy()\n", "renamed[\"Trigger Type\"] = renamed[\"type\"].apply(\n", - " lambda x: f\"Amount \\n[{amount_policies} s.]\" if x == \"amount\" else f\"Time [{time_policies} yrs.]\"\n", + " lambda x: f\"Amount \\n[{amount_policies}]\" if x == \"amount\" else f\"Time [{time_policies} yrs.]\"\n", ")\n", "renamed" ] @@ -412,6 +415,13 @@ "save_plot(fig, \"tradeoff_simple_arxiv_triggers_cost\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, diff --git a/analytics/plotting/rh_thesis/triggers_simple/hp_cost_perf_tradeoff.ipynb b/analytics/plotting/rh_thesis/triggers_simple/hp_cost_perf_tradeoff.ipynb index ecf1ef157..3a2b9325f 100644 --- a/analytics/plotting/rh_thesis/triggers_simple/hp_cost_perf_tradeoff.ipynb +++ b/analytics/plotting/rh_thesis/triggers_simple/hp_cost_perf_tradeoff.ipynb @@ -331,7 +331,10 @@ "source": [ "# extract number if samples from pipeline name\n", "amount_policies = \"/\".join(\n", - " [str(x) for x in [str(y).replace(\" samples\", \"\") for y in merged[merged[\"type\"] == \"amount\"][\"pipeline_ref\"]]]\n", + " [\n", + " f\"{int(x) // 1000}k\"\n", + " for x in [str(y).replace(\" samples\", \"\") for y in merged[merged[\"type\"] == \"amount\"][\"pipeline_ref\"]]\n", + " ]\n", ")\n", "time_policies = \"/\".join(\n", " [\n", @@ -347,7 +350,7 @@ "\n", "renamed = merged.copy()\n", "renamed[\"Trigger Type\"] = renamed[\"type\"].apply(\n", - " lambda x: f\"Amount \\n[{amount_policies} s.]\" if x == \"amount\" else f\"Time [{time_policies} yrs.]\"\n", + " lambda x: f\"Amount \\n[{amount_policies}]\" if x == \"amount\" else f\"Time [{time_policies} yrs.]\"\n", ")\n", "renamed" ] @@ -367,7 +370,7 @@ " x_label=\"Number of Triggers\",\n", " y_label=\"Mean Accuracy %\",\n", " height_factor=0.6,\n", - " width_factor=0.65,\n", + " width_factor=0.7,\n", ")\n", "\n", "save_plot(fig, \"tradeoff_simple_huffpost_triggers_performance\")" @@ -391,7 +394,7 @@ " x_label=\"Total Cost (Minutes)\",\n", " y_label=\"Mean Accuracy %\",\n", " height_factor=0.6,\n", - " width_factor=0.65,\n", + " width_factor=0.7,\n", ")\n", "\n", "save_plot(fig, \"tradeoff_simple_huffpost_cost_performance\")" @@ -417,6 +420,13 @@ "\n", "save_plot(fig, \"tradeoff_simple_huffpost_triggers_cost\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/analytics/plotting/rh_thesis/triggers_simple/yb_cost_perf_tradeoff.ipynb b/analytics/plotting/rh_thesis/triggers_simple/yb_cost_perf_tradeoff.ipynb index 9687efb85..6613ffcb2 100644 --- a/analytics/plotting/rh_thesis/triggers_simple/yb_cost_perf_tradeoff.ipynb +++ b/analytics/plotting/rh_thesis/triggers_simple/yb_cost_perf_tradeoff.ipynb @@ -279,7 +279,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_triggers = df_leaf[df_leaf[\"id\"] == PipelineStage.TRAIN.name]" + "df_leaf[\"id\"].unique()" ] }, { @@ -350,14 +350,20 @@ " [\n", " str(x)\n", " for x in list(\n", - " sorted(merged[merged[\"type\"] == \"amount\"][\"pipeline_ref\"].str.extract(r\"(\\d+) .*\").astype(int)[0])\n", + " sorted(\n", + " merged[merged[\"type\"] == \"amount\"][\"pipeline_ref\"].str.extract(r\"(\\d+) .*\").astype(int)[0], reverse=True\n", + " )\n", " )\n", " ]\n", ")\n", "time_policies = \"/\".join(\n", " [\n", " str(x)\n", - " for x in list(sorted(merged[merged[\"type\"] == \"time\"][\"pipeline_ref\"].str.extract(r\"(\\d+) .*\").astype(int)[0]))\n", + " for x in list(\n", + " sorted(\n", + " merged[merged[\"type\"] == \"time\"][\"pipeline_ref\"].str.extract(r\"(\\d+) .*\").astype(int)[0], reverse=True\n", + " )\n", + " )\n", " ]\n", ")\n", "print(amount_policies)\n", @@ -435,6 +441,13 @@ "\n", "save_plot(fig, \"tradeoff_simple_yearbook_triggers_cost\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {