Skip to content

Commit

Permalink
fix_issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@elephaint
elephaint committed Oct 8, 2024
1 parent 079a804 commit 4561ef1
Show file tree
Hide file tree
Showing 6 changed files with 350 additions and 279 deletions.
167 changes: 48 additions & 119 deletions nbs/core.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@
" TimeMixer, KAN, RMoK\n",
")\n",
"from neuralforecast.common._base_auto import BaseAuto, MockTrial\n",
"from neuralforecast.utils import ConformalIntervals, get_conformal_method"
"from neuralforecast.utils import PredictionIntervals, get_prediction_interval_method"
]
},
{
Expand Down Expand Up @@ -507,7 +507,7 @@
" time_col: str = 'ds',\n",
" target_col: str = 'y',\n",
" distributed_config: Optional[DistributedConfig] = None,\n",
" conformal_intervals: Optional[ConformalIntervals] = None,\n",
" prediction_intervals: Optional[PredictionIntervals] = None,\n",
" ) -> None:\n",
" \"\"\"Fit the core.NeuralForecast.\n",
"\n",
Expand Down Expand Up @@ -537,7 +537,7 @@
" Column that contains the target.\n",
" distributed_config : neuralforecast.DistributedConfig\n",
" Configuration to use for DDP training. Currently only spark is supported.\n",
" conformal_intervals : ConformalIntervals, optional (default=None)\n",
" prediction_intervals : PredictionIntervals, optional (default=None)\n",
" Configuration to calibrate prediction intervals (Conformal Prediction). \n",
"\n",
" Returns\n",
Expand All @@ -556,7 +556,7 @@
" raise Exception('Set val_size>0 if early stopping is enabled.')\n",
" \n",
" self._cs_df: Optional[DFType] = None\n",
" self.conformal_intervals: Optional[ConformalIntervals] = None\n",
" self.prediction_intervals: Optional[PredictionIntervals] = None\n",
"\n",
" # Process and save new dataset (in self)\n",
" if isinstance(df, (pd.DataFrame, pl_DataFrame)):\n",
Expand Down Expand Up @@ -610,9 +610,8 @@
" if self.dataset.min_size < val_size:\n",
" warnings.warn('Validation set size is larger than the shorter time-series.')\n",
"\n",
" if conformal_intervals is not None:\n",
" # conformal prediction\n",
" self.conformal_intervals = conformal_intervals\n",
" if prediction_intervals is not None:\n",
" self.prediction_intervals = prediction_intervals\n",
" self._cs_df = self._conformity_scores(\n",
" df=df,\n",
" id_col=id_col,\n",
Expand Down Expand Up @@ -726,12 +725,11 @@
"\n",
" return futr_exog | set(hist_exog)\n",
" \n",
" def _get_model_names(self, conformal=False, enable_quantiles=False) -> List[str]:\n",
" def _get_model_names(self, add_level=False) -> List[str]:\n",
" names: List[str] = []\n",
" count_names = {'model': 0}\n",
" for model in self.models:\n",
" if conformal and not enable_quantiles and model.loss.outputsize_multiplier > 1:\n",
" # skip prediction intervals on quantile outputs\n",
" if add_level and model.loss.outputsize_multiplier > 1:\n",
" continue\n",
"\n",
" model_name = repr(model)\n",
Expand Down Expand Up @@ -856,7 +854,7 @@
" sort_df: bool = True,\n",
" verbose: bool = False,\n",
" engine = None,\n",
" conformal_level: Optional[List[Union[int, float]]] = None,\n",
" level: Optional[List[Union[int, float]]] = None,\n",
" **data_kwargs\n",
" ):\n",
" \"\"\"Predict with core.NeuralForecast.\n",
Expand All @@ -878,8 +876,8 @@
" Print processing steps.\n",
" engine : spark session\n",
" Distributed engine for inference. Only used if df is a spark dataframe or if fit was called on a spark dataframe.\n",
" conformal_level : list of ints or floats, optional (default=None)\n",
" Confidence levels between 0 and 100 for conformal intervals.\n",
" level : list of ints or floats, optional (default=None)\n",
" Confidence levels between 0 and 100.\n",
" data_kwargs : kwargs\n",
" Extra arguments to be passed to the dataset within each model.\n",
"\n",
Expand Down Expand Up @@ -1015,24 +1013,21 @@
" _warn_id_as_idx()\n",
" fcsts_df = fcsts_df.set_index(self.id_col)\n",
"\n",
" # perform conformal predictions\n",
" if conformal_level is not None:\n",
" if self._cs_df is None or self.conformal_intervals is None:\n",
" warn_msg = (\n",
" 'Please rerun the `fit` method passing a valid conformal_interval settings to compute conformity scores'\n",
" )\n",
" warnings.warn(warn_msg, UserWarning)\n",
" # add prediction intervals\n",
" if level is not None:\n",
" if self._cs_df is None or self.prediction_intervals is None:\n",
" raise Exception('You must fit the model with prediction_intervals to use level.')\n",
" else:\n",
" level_ = sorted(conformal_level)\n",
" model_names = self._get_model_names(conformal=True, enable_quantiles=self.conformal_intervals.enable_quantiles)\n",
" conformal_method = get_conformal_method(self.conformal_intervals.method)\n",
" level_ = sorted(level)\n",
" model_names = self._get_model_names(add_level=True)\n",
" prediction_interval_method = get_prediction_interval_method(self.prediction_intervals.method)\n",
"\n",
" fcsts_df = conformal_method(\n",
" fcsts_df = prediction_interval_method(\n",
" fcsts_df,\n",
" self._cs_df,\n",
" model_names=list(model_names),\n",
" level=level_,\n",
" cs_n_windows=self.conformal_intervals.n_windows,\n",
" cs_n_windows=self.prediction_intervals.n_windows,\n",
" n_series=len(uids),\n",
" horizon=self.h,\n",
" )\n",
Expand Down Expand Up @@ -1522,8 +1517,7 @@
" \"id_col\": self.id_col,\n",
" \"time_col\": self.time_col,\n",
" \"target_col\": self.target_col,\n",
" # conformal prediction\n",
" \"conformal_intervals\": self.conformal_intervals,\n",
" \"prediction_intervals\": self.prediction_intervals,\n",
" \"_cs_df\": self._cs_df, # conformity score\n",
" }\n",
" if save_dataset:\n",
Expand Down Expand Up @@ -1613,7 +1607,7 @@
" for attr in ['id_col', 'time_col', 'target_col']:\n",
" setattr(neuralforecast, attr, config_dict[attr])\n",
" # only restore attribute if available\n",
" for attr in ['conformal_intervals', '_cs_df']:\n",
" for attr in ['prediction_intervals', '_cs_df']:\n",
" if attr in config_dict.keys():\n",
" setattr(neuralforecast, attr, config_dict[attr])\n",
"\n",
Expand Down Expand Up @@ -1647,21 +1641,21 @@
" \"\"\"Compute conformity scores.\n",
" \n",
" We need at least two cross validation errors to compute\n",
" quantiles for prediction intervals (`n_windows=2`, specified by self.conformal_intervals).\n",
" quantiles for prediction intervals (`n_windows=2`, specified by self.prediction_intervals).\n",
" \n",
" The exception is raised by the ConformalIntervals data class.\n",
" The exception is raised by the PredictionIntervals data class.\n",
"\n",
" df: Optional[Union[DataFrame, SparkDataFrame, Sequence[str]]] = None,\n",
" id_col: str = 'unique_id',\n",
" time_col: str = 'ds',\n",
" target_col: str = 'y',\n",
" static_df: Optional[Union[DataFrame, SparkDataFrame]] = None,\n",
" \"\"\"\n",
" if self.conformal_intervals is None:\n",
" raise AttributeError('Please rerun the `fit` method passing a valid conformal_interval settings to compute conformity scores')\n",
" if self.prediction_intervals is None:\n",
" raise AttributeError('Please rerun the `fit` method passing a valid prediction_interval setting to compute conformity scores')\n",
" \n",
" min_size = ufp.counts_by_id(df, id_col)['counts'].min()\n",
" min_samples = self.h * self.conformal_intervals.n_windows + 1\n",
" min_samples = self.h * self.prediction_intervals.n_windows + 1\n",
" if min_size < min_samples:\n",
" raise ValueError(\n",
" \"Minimum required samples in each serie for the prediction intervals \"\n",
Expand All @@ -1672,15 +1666,15 @@
" cv_results = self.cross_validation(\n",
" df=df,\n",
" static_df=static_df,\n",
" n_windows=self.conformal_intervals.n_windows,\n",
" n_windows=self.prediction_intervals.n_windows,\n",
" id_col=id_col,\n",
" time_col=time_col,\n",
" target_col=target_col,\n",
" )\n",
" \n",
" kept = [time_col, id_col, 'cutoff']\n",
" # conformity score for each model\n",
" for model in self._get_model_names(conformal=True, enable_quantiles=self.conformal_intervals.enable_quantiles):\n",
" for model in self._get_model_names(add_level=True):\n",
" kept.append(model)\n",
"\n",
" # compute absolute error for each model\n",
Expand Down Expand Up @@ -2622,9 +2616,9 @@
" ],\n",
" freq='M'\n",
")\n",
"conformal_intervals = ConformalIntervals()\n",
"fcst.fit(AirPassengersPanel_train, conformal_intervals=conformal_intervals)\n",
"forecasts1 = fcst.predict(futr_df=AirPassengersPanel_test, conformal_level=[50])\n",
"prediction_intervals = PredictionIntervals()\n",
"fcst.fit(AirPassengersPanel_train, prediction_intervals=prediction_intervals)\n",
"forecasts1 = fcst.predict(futr_df=AirPassengersPanel_test, level=[50])\n",
"save_paths = ['./examples/debug_run/']\n",
"try:\n",
" s3fs.S3FileSystem().ls('s3://nixtla-tmp') \n",
Expand All @@ -2638,7 +2632,7 @@
"for path in save_paths:\n",
" fcst.save(path=path, model_index=None, overwrite=True, save_dataset=True)\n",
" fcst2 = NeuralForecast.load(path=path)\n",
" forecasts2 = fcst2.predict(futr_df=AirPassengersPanel_test, conformal_level=[50])\n",
" forecasts2 = fcst2.predict(futr_df=AirPassengersPanel_test, level=[50])\n",
" pd.testing.assert_frame_equal(forecasts1, forecasts2[forecasts1.columns])"
]
},
Expand Down Expand Up @@ -3332,19 +3326,19 @@
"#| hide\n",
"# test conformal prediction, method=conformal_distribution\n",
"\n",
"conformal_intervals = ConformalIntervals()\n",
"prediction_intervals = PredictionIntervals()\n",
"\n",
"models = []\n",
"for nf_model in [NHITS, RNN, StemGNN]:\n",
"for nf_model in [NHITS, RNN, TSMixer]:\n",
" params = {\"h\": 12, \"input_size\": 24, \"max_steps\": 1}\n",
" if nf_model.__name__ == \"StemGNN\":\n",
" if nf_model.__name__ == \"TSMixer\":\n",
" params.update({\"n_series\": 2})\n",
" models.append(nf_model(**params))\n",
"\n",
"\n",
"nf = NeuralForecast(models=models, freq='M')\n",
"nf.fit(AirPassengersPanel_train, conformal_intervals=conformal_intervals)\n",
"preds = nf.predict(futr_df=AirPassengersPanel_test, conformal_level=[10, 50, 90])"
"nf.fit(AirPassengersPanel_train, prediction_intervals=prediction_intervals)\n",
"preds = nf.predict(futr_df=AirPassengersPanel_test, level=[90])"
]
},
{
Expand All @@ -3358,19 +3352,19 @@
"#| polars\n",
"# test conformal prediction works for polar dataframe\n",
"\n",
"conformal_intervals = ConformalIntervals()\n",
"prediction_intervals = PredictionIntervals()\n",
"\n",
"models = []\n",
"for nf_model in [NHITS, RNN, StemGNN]:\n",
"for nf_model in [NHITS, RNN, TSMixer]:\n",
" params = {\"h\": 12, \"input_size\": 24, \"max_steps\": 1}\n",
" if nf_model.__name__ == \"StemGNN\":\n",
" if nf_model.__name__ == \"TSMixer\":\n",
" params.update({\"n_series\": 2})\n",
" models.append(nf_model(**params))\n",
"\n",
"\n",
"nf = NeuralForecast(models=models, freq='1mo')\n",
"nf.fit(AirPassengers_pl, conformal_intervals=conformal_intervals, time_col='time', id_col='uid', target_col='target')\n",
"preds = nf.predict(conformal_level=[10, 50, 90])"
"nf.fit(AirPassengers_pl, prediction_intervals=prediction_intervals, time_col='time', id_col='uid', target_col='target')\n",
"preds = nf.predict(level=[90])"
]
},
{
Expand All @@ -3383,84 +3377,19 @@
"#| hide\n",
"# test conformal prediction, method=conformal_error\n",
"\n",
"conformal_intervals = ConformalIntervals(method=\"conformal_error\")\n",
"prediction_intervals = PredictionIntervals(method=\"conformal_error\")\n",
"\n",
"models = []\n",
"for nf_model in [NHITS, RNN, StemGNN]:\n",
"for nf_model in [NHITS, RNN, TSMixer]:\n",
" params = {\"h\": 12, \"input_size\": 24, \"max_steps\": 1}\n",
" if nf_model.__name__ == \"StemGNN\":\n",
" if nf_model.__name__ == \"TSMixer\":\n",
" params.update({\"n_series\": 2})\n",
" models.append(nf_model(**params))\n",
"\n",
"\n",
"nf = NeuralForecast(models=models, freq='M')\n",
"nf.fit(AirPassengersPanel_train, conformal_intervals=conformal_intervals)\n",
"preds = nf.predict(futr_df=AirPassengersPanel_test, conformal_level=[10, 50, 90])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d25b2cd2",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"# test conformal prediction are not applied for models with quantiled-related loss\n",
"# by default (ConformalIntervals.enable_quantiles=False)\n",
"\n",
"conformal_intervals = ConformalIntervals()\n",
"\n",
"models = []\n",
"for nf_model in [NHITS, RNN]:\n",
" params = {\"h\": 12, \"input_size\": 24, \"max_steps\": 1}\n",
" if nf_model.__name__ == \"NHITS\":\n",
" params.update({\"loss\": MQLoss(level=[80])})\n",
" models.append(nf_model(**params))\n",
"\n",
"\n",
"nf = NeuralForecast(models=models, freq='M')\n",
"nf.fit(AirPassengersPanel_train, conformal_intervals=conformal_intervals)\n",
"preds = nf.predict(futr_df=AirPassengersPanel_test, conformal_level=[10, 50, 90])\n",
"\n",
"pred_cols = [\n",
" 'NHITS-median', 'NHITS-lo-80', 'NHITS-hi-80', 'RNN',\n",
" 'RNN-conformal-lo-90', 'RNN-conformal-lo-50', 'RNN-conformal-lo-10',\n",
" 'RNN-conformal-hi-10', 'RNN-conformal-hi-50', 'RNN-conformal-hi-90'\n",
"]\n",
"assert all([col in preds.columns for col in pred_cols])\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7b980087",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"# test conformal predictions applied to quantiles if ConformalIntervals.enable_quantiles=True\n",
"\n",
"conformal_intervals = ConformalIntervals(enable_quantiles=True)\n",
"\n",
"nf = NeuralForecast(models=[NHITS(h=12, input_size=24, max_steps=1, loss=MQLoss(level=[80]))], freq='M')\n",
"nf.fit(AirPassengersPanel_train, conformal_intervals=conformal_intervals)\n",
"preds = nf.predict(futr_df=AirPassengersPanel_test, conformal_level=[10, 50, 90])\n",
"\n",
"pred_cols = [\n",
" 'NHITS-median', 'NHITS-lo-80', 'NHITS-hi-80',\n",
" 'NHITS-median-conformal-lo-90', 'NHITS-median-conformal-lo-50',\n",
" 'NHITS-median-conformal-lo-10', 'NHITS-median-conformal-hi-10',\n",
" 'NHITS-median-conformal-hi-50', 'NHITS-median-conformal-hi-90',\n",
" 'NHITS-lo-80-conformal-lo-90', 'NHITS-lo-80-conformal-lo-50',\n",
" 'NHITS-lo-80-conformal-lo-10', 'NHITS-lo-80-conformal-hi-10',\n",
" 'NHITS-lo-80-conformal-hi-50', 'NHITS-lo-80-conformal-hi-90',\n",
" 'NHITS-hi-80-conformal-lo-90', 'NHITS-hi-80-conformal-lo-50',\n",
" 'NHITS-hi-80-conformal-lo-10', 'NHITS-hi-80-conformal-hi-10',\n",
" 'NHITS-hi-80-conformal-hi-50', 'NHITS-hi-80-conformal-hi-90'\n",
"]\n",
"\n",
"assert all([col in preds.columns for col in pred_cols])\n"
"nf.fit(AirPassengersPanel_train, prediction_intervals=prediction_intervals)\n",
"preds = nf.predict(futr_df=AirPassengersPanel_test, level=[90])"
]
}
],
Expand Down
Loading

0 comments on commit 4561ef1

Please sign in to comment.