Teach OutliersTransform to ignore holidays

CHANGELOG.md

@@ -38,7 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - 
 - 
 - 
-- 
+- Add in `OutliersTransform` possibilities use `ignore_flag_column` to skip values use ignore ([#291](https://github.com/etna-team/etna/pull/291))
 
 ### Changed
 - 
@@ -98,7 +98,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - 
 - 
 - 
-- 
+- Fix `PredictionIntervalOutliersTransform` fails to work with created columns ([#293](https://github.com/etna-team/etna/issues/293))
 - Prohibit empty list value and duplication of `target_timestamps` parameter in `FoldMask` ([#226](https://github.com/etna-team/etna/pull/226))
 - 
 - 

etna/analysis/outliers/prediction_interval_outliers.py

@@ -50,13 +50,17 @@ def _select_segments_subset(ts: TSDataset, segments: List[str]) -> TSDataset:
     result: TSDataset
         dataset with selected column.
     """
-    df = ts.raw_df.loc[:, pd.IndexSlice[segments, :]].copy()
+    df = ts.df.loc[:, pd.IndexSlice[segments, :]].copy()
     df = df.dropna()
     df_exog = ts.df_exog
     if df_exog is not None:
         df_exog = df_exog.loc[:, pd.IndexSlice[segments, :]].copy()
     known_future = ts.known_future
     freq = ts.freq
+
+    if df_exog is not None:
+        df = df.drop(df_exog.columns.get_level_values("feature").values.tolist(), axis=1, level=1)
+
     subset_ts = TSDataset(df=df, df_exog=df_exog, known_future=known_future, freq=freq)
     return subset_ts
 

etna/transforms/outliers/base.py

@@ -17,17 +17,24 @@
 class OutliersTransform(ReversibleTransform, ABC):
     """Finds outliers in specific columns of DataFrame and replaces it with NaNs."""
 
-    def __init__(self, in_column: str):
+    def __init__(self, in_column: str, ignore_flag_column: Optional[str] = None):
         """
         Create instance of OutliersTransform.
 
         Parameters
         ----------
         in_column:
             name of processed column
+        ignore_flag_column:
+            column name for skipping values from outlier check
         """
-        super().__init__(required_features=[in_column])
+        required_features = [in_column]
+        if ignore_flag_column:
+            required_features.append(ignore_flag_column)
+
+        super().__init__(required_features=required_features)
         self.in_column = in_column
+        self.ignore_flag_column = ignore_flag_column
 
         self.segment_outliers: Optional[Dict[str, pd.Series]] = None
 
@@ -78,6 +85,15 @@
         :
             The fitted transform instance.
         """
+        if self.ignore_flag_column is not None:
+            if self.ignore_flag_column not in ts.columns.get_level_values("feature"):
+                raise ValueError(f'Name ignore_flag_column="{self.ignore_flag_column}" not find.')
+            types_ignore_flag = ts[..., self.ignore_flag_column].isin([0, 1]).all(axis=0)
+            if not all(types_ignore_flag):
+                raise ValueError(
+                    f'Columns ignore_flag contain non binary value: columns: "{self.ignore_flag_column}" in segment: {types_ignore_flag[~types_ignore_flag].index.get_level_values("segment").tolist()}'
+                )
+
         self.segment_outliers = self.detect_outliers(ts)
         self._fit_segments = ts.segments
         super().fit(ts=ts)
@@ -131,8 +147,16 @@
             if segment not in segments:
                 continue
             # to locate only present indices
-            segment_outliers_timestamps = list(index_set.intersection(self.segment_outliers[segment].index.values))
+            if self.ignore_flag_column:
+                available_points = set(df[df[segment, self.ignore_flag_column] == 0].index.values)
+            else:
+                available_points = index_set
+            segment_outliers_timestamps = list(
+                available_points.intersection(self.segment_outliers[segment].index.values)
+            )
+
             df.loc[segment_outliers_timestamps, pd.IndexSlice[segment, self.in_column]] = np.NaN
+
         return df
 
     def _inverse_transform(self, df: pd.DataFrame) -> pd.DataFrame:

etna/transforms/outliers/point_outliers.py

@@ -1,6 +1,7 @@
 from typing import Callable
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Type
 from typing import Union
 
@@ -32,7 +33,13 @@ class MedianOutliersTransform(OutliersTransform):
     it uses information from the whole train part.
     """
 
-    def __init__(self, in_column: str, window_size: int = 10, alpha: float = 3):
+    def __init__(
+        self,
+        in_column: str,
+        window_size: int = 10,
+        alpha: float = 3,
+        ignore_flag_column: Optional[str] = None,
+    ):
         """Create instance of MedianOutliersTransform.
 
         Parameters
@@ -43,10 +50,12 @@ def __init__(self, in_column: str, window_size: int = 10, alpha: float = 3):
             number of points in the window
         alpha:
             coefficient for determining the threshold
+        ignore_flag_column:
+            column name for skipping values from outlier check
         """
         self.window_size = window_size
         self.alpha = alpha
-        super().__init__(in_column=in_column)
+        super().__init__(in_column=in_column, ignore_flag_column=ignore_flag_column)
 
     def detect_outliers(self, ts: TSDataset) -> Dict[str, List[pd.Timestamp]]:
         """Call :py:func:`~etna.analysis.outliers.median_outliers.get_anomalies_median` function with self parameters.
@@ -97,6 +106,7 @@ def __init__(
         distance_coef: float = 3,
         n_neighbors: int = 3,
         distance_func: Union[Literal["absolute_difference"], Callable[[float, float], float]] = "absolute_difference",
+        ignore_flag_column: Optional[str] = None,
     ):
         """Create instance of DensityOutliersTransform.
 
@@ -113,12 +123,14 @@ def __init__(
         distance_func:
             distance function. If a string is specified, a corresponding vectorized implementation will be used.
             Custom callable will be used as a scalar function, which will result in worse performance.
+        ignore_flag_column:
+            column name for skipping values from outlier check
         """
         self.window_size = window_size
         self.distance_coef = distance_coef
         self.n_neighbors = n_neighbors
         self.distance_func = distance_func
-        super().__init__(in_column=in_column)
+        super().__init__(in_column=in_column, ignore_flag_column=ignore_flag_column)
 
     def detect_outliers(self, ts: TSDataset) -> Dict[str, List[pd.Timestamp]]:
         """Call :py:func:`~etna.analysis.outliers.density_outliers.get_anomalies_density` function with self parameters.
@@ -169,6 +181,7 @@ def __init__(
         in_column: str,
         model: Union[Literal["prophet"], Literal["sarimax"], Type["ProphetModel"], Type["SARIMAXModel"]],
         interval_width: float = 0.95,
+        ignore_flag_column: Optional[str] = None,
         **model_kwargs,
     ):
         """Create instance of PredictionIntervalOutliersTransform.
@@ -181,7 +194,8 @@ def __init__(
             model for prediction interval estimation
         interval_width:
             width of the prediction interval
-
+        ignore_flag_column:
+            column name for skipping values from outlier check
         Notes
         -----
         For not "target" column only column data will be used for learning.
@@ -190,7 +204,7 @@ def __init__(
         self.interval_width = interval_width
         self.model_kwargs = model_kwargs
         self._model_type = self._get_model_type(model)
-        super().__init__(in_column=in_column)
+        super().__init__(in_column=in_column, ignore_flag_column=ignore_flag_column)
 
     @staticmethod
     def _get_model_type(