Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OCTO-1143 Review outlier filtering #59

Merged
merged 6 commits into from
Dec 8, 2016
Merged

OCTO-1143 Review outlier filtering #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1ab9e5a
OCTO-1026 removed output of empty outlier filter rules in metadata
domheger Oct 31, 2016
8aa1e43
OCTO-1026 added minimum scaling of thresholds according to time inter…
domheger Nov 1, 2016
84cb1d5
OCTO-1026 added minimum scaling of thresholds according to time inter…
domheger Nov 1, 2016
119c5a5
Merge branch 'feature/OCTO-1026-outlier-filtering' of github.com:zala…
domheger Nov 2, 2016
b3f61cc
Change KeyError to ValueError; Turn off duplicates sanity check for t…
Dec 8, 2016
50e33fe
OCTO-1183 adapted and added tests for outlier filtering with scaling …
domheger Dec 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions expan/core/experimentdata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ def __init__(self, metrics=None, metadata={}, features='default',
self.metadata['primary_KPI']))

if len(self.variant_names) < 2:
raise KeyError('Less than 2 variants found!')
raise ValueError('Less than 2 variants found!')

self.features.set_index(list(feature_indices), inplace=True)
self.kpis.set_index(list(kpi_indices), inplace=True)
Expand All @@ -137,6 +137,8 @@ def __init__(self, metrics=None, metadata={}, features='default',
# appropriate
self.kpis = self.kpis_time.groupby(level=['entity', 'variant']).sum()
else:
if self.kpis.reset_index()['entity'].nunique()<len(self.kpis):
raise ValueError("Column 'entity' is not unique!")
self.kpis_time = None

@property
Expand Down Expand Up @@ -239,6 +241,7 @@ def _filter_threshold(self, params, drop_thresh_column=True):
int: number of entities filtered out
"""
used_rule = {}
is_outlier=[]

if 'metric' in params and 'value' in params: #and not ('time_interval' in params and not 'treatment_stop_time' in params):
# if the time interval is set calculate a linearly adjusted threshold and store it in a separate column
Expand All @@ -247,17 +250,19 @@ def _filter_threshold(self, params, drop_thresh_column=True):
# start timestamp exists as a feature
# NOTE: treatment_start_time and treatment_exposure have to be epoch time in seconds
if 'treatment_start_time' in self.features.columns and 'treatment_stop_time' in params:
self.kpis = self.kpis.assign(calc_thresh_value = lambda x: (params['value'] * ((params['treatment_stop_time'] - self.features['treatment_start_time']) / params['time_interval'])), axis='rows')
# set minimum scaling to time_interval defined in rule
scale_factors=np.maximum( (params['treatment_stop_time'] - self.features['treatment_start_time']) / params['time_interval'], 1)
self.kpis = self.kpis.assign(calc_thresh_value = lambda x: (params['value'] * scale_factors), axis='rows')
# treatment exposure exists as a feature
elif 'treatment_exposure' in self.features.columns:
self.kpis['calc_thresh_value'] = params['value'] * self.features.treatment_exposure / params['time_interval']
scale_factors=np.maximum( self.features.treatment_exposure / params['time_interval'], 1)
self.kpis['calc_thresh_value'] = params['value'] * scale_factors
else:
warnings.warn('Scaling by time not possible, using hard threshold instead!')
self.kpis['calc_thresh_value'] = params['value']
else:
self.kpis['calc_thresh_value'] = params['value']

is_outlier=[]
if params['kind'] == 'lower':
if params['metric'] in self.kpis.columns:
is_outlier = self.kpis[params['metric']] < self.kpis.calc_thresh_value
Expand Down Expand Up @@ -327,7 +332,10 @@ def filter_outliers(self, rules, drop_thresh=True):
Given these parameters a per entity threshold is calculated by the following equation:

.. math::
threshold = value * \\frac{treatment\_stop\_time - treatment\_start\_time}{time\_interval}
threshold = value * min( \\frac{treatment\_stop\_time - treatment\_start\_time}{time\_interval} , 1)

Using the equation above, the threshold defined for a specific time_interval is scaled according to the treatment exposure time for each entity.
Thereby, the minimum scaling factor is 1, i.e. the time_interval defined in the outlier rules is the smallest threshold that is applied.

>>>
[
Expand All @@ -353,8 +361,9 @@ def filter_outliers(self, rules, drop_thresh=True):
for rule in rules:
if rule['type'] == 'threshold':
urule, n = self._filter_threshold(params=rule, drop_thresh_column=drop_thresh)
used_rules.append(urule)
n_filtered.append(n)
if n > 0:
used_rules.append(urule)
n_filtered.append(n)

# store rules in the metadata
self.metadata['outlier_filter'] = used_rules
Expand Down Expand Up @@ -400,7 +409,6 @@ def detect_features(metrics):
"type":"threshold",
"value": -1.0,
"kind": "lower",
"time_interval": 30758400,
#"treatment_stop_time": 30758500
"time_interval": 30758400
}
])
44 changes: 34 additions & 10 deletions tests/tests_core/test_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,12 @@ def tearDown(self):

def test_create_with_insufficient_data(self):
# should not work:
with self.assertRaises(KeyError):
with self.assertRaises(ValueError):
ExperimentData(
pd.DataFrame(columns=['entity', 'variant']),
metadata={'experiment': 'test', 'source': 'none'}
)
with self.assertRaises(KeyError):
with self.assertRaises(ValueError):
ExperimentData(
pd.DataFrame(columns=['entity', 'variant', 'plums']),
metadata={'experiment': 'test', 'source': 'none', 'primary_KPI': 'plums'}
Expand All @@ -121,7 +121,7 @@ def test_create_with_insufficient_data(self):
# pd.DataFrame(columns=['entity', 'treatment_start']),
# metadata={'experiment': 'fesf', 'source': 'random'},
# )
with self.assertRaises(KeyError):
with self.assertRaises(ValueError):
ExperimentData(
pd.DataFrame(columns=['variant', 'treatment_start']),
metadata={'experiment': 'fesf', 'source': 'random'}
Expand Down Expand Up @@ -220,7 +220,7 @@ def test_outlier_filtering(self):
metrics_outlier.loc[idx, "normal_shifted_by_feature"] += np.sign(metrics_outlier.loc[idx, "normal_shifted_by_feature"])
metrics_outlier.loc[idx, "normal_shifted_by_feature"] *= 10

# use 4 rules, one is not implemented, default settings
# use 4 rules, one is not implemented and one does not apply, default settings
D = ExperimentData(metrics=metrics_outlier, metadata=self.metadata)
D.filter_outliers(rules=[{"metric":"normal_shifted_by_feature",
"type":"threshold",
Expand All @@ -233,17 +233,17 @@ def test_outlier_filtering(self):
"kind": "upper"
},
{"metric": "normal_same",
"type": "threshold",
"type": "threshold", #this does not apply
"value": 10.0,
"kind": "upper"
},
{"metric": "normal_same",
"type": "water",
"type": "water", #this is not implemented
"value": 10.0,
"kind": "both"
}
])
self.assertEqual(len(D.metadata['outlier_filter']), 3)
self.assertEqual(len(D.metadata['outlier_filter']), 2) # only are actually applied
self.assertEqual(len(D.metrics), 9000)
for i in idx:
self.assertEqual(D.metrics.ix[i].empty, True)
Expand Down Expand Up @@ -311,16 +311,40 @@ def test_outlier_filtering_n_filtered(self):

def test_outlier_filtering_treatment_exposure(self):
"""Check if scaling of the threshold works when the treatment_exposure is provided"""
self.metrics['treatment_exposure'] = self.metrics['treatment_start_time']
self.metrics['treatment_exposure'] = 1000
D = ExperimentData(self.metrics[['entity','variant','normal_shifted','treatment_exposure']], self.metadata, features=[3])
D.filter_outliers(rules=[{"metric":"normal_shifted",
"type":"threshold",
"value": -1.0,
"kind": "lower",
"time_interval": 1000
}
])
n_filtered=D.metadata['n_filtered']
self.assertEqual(n_filtered, [1082])

#setting a larger long time_interval than the treatment_exposure does not affect the scaling of the threshold
D = ExperimentData(self.metrics[['entity','variant','normal_shifted','treatment_exposure']], self.metadata, features=[3])
D.filter_outliers(rules=[{"metric":"normal_shifted",
"type":"threshold",
"value": -1.0,
"kind": "lower",
"time_interval": 10000
}
])
self.assertEqual(D.metadata['n_filtered'], D.metadata['n_filtered'])

#setting a lower time_interval filters less, here so much less that filter will not be applied
D = ExperimentData(self.metrics[['entity','variant','normal_shifted','treatment_exposure']], self.metadata, features=[3])
D.filter_outliers(rules=[{"metric":"normal_shifted",
"type":"threshold",
"value": -1.0,
"kind": "lower",
"time_interval": 30758400
"time_interval": 100
}
])
self.assertEqual(D.metadata['n_filtered'], [3695])
self.assertEqual(len(D.metadata['outlier_filter']), 0)
self.assertEqual(len(D.metadata['n_filtered']), 0)

if __name__ == '__main__':
unittest.main()