Skip to content

Commit

Permalink
fix(experiments): Fix flaky trends query runner test (#26898)
Browse files Browse the repository at this point in the history
  • Loading branch information
@danielbachhuber
danielbachhuber authored Dec 13, 2024
1 parent 646de76 commit 37c6b41
Showing 1 changed file with 120 additions and 20 deletions.
140 changes: 120 additions & 20 deletions posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,11 @@ def test_query_runner(self):
team=self.team,
event="$feature_flag_called",
distinct_id=f"user_{variant}_{i}",
properties={feature_flag_property: variant},
properties={
"$feature_flag_response": variant,
feature_flag_property: variant,
"$feature_flag": feature_flag.key,
},
)

flush_persons_and_events()
Expand Down Expand Up @@ -530,7 +534,11 @@ def test_query_runner_with_holdout(self):
team=self.team,
event="$feature_flag_called",
distinct_id=f"user_{variant}_{i}",
properties={feature_flag_property: variant},
properties={
"$feature_flag_response": variant,
feature_flag_property: variant,
"$feature_flag": feature_flag.key,
},
)

flush_persons_and_events()
Expand Down Expand Up @@ -705,7 +713,11 @@ def test_query_runner_with_data_warehouse_series_avg_amount(self):
team=self.team,
event="$feature_flag_called",
distinct_id=f"user_{variant}_{i}",
properties={feature_flag_property: variant},
properties={
"$feature_flag_response": variant,
feature_flag_property: variant,
"$feature_flag": feature_flag.key,
},
timestamp=datetime(2023, 1, i + 1),
)

Expand All @@ -719,21 +731,33 @@ def test_query_runner_with_data_warehouse_series_avg_amount(self):
team=self.team,
event="$feature_flag_called",
distinct_id="user_test_3",
properties={feature_flag_property: "control"},
properties={
"$feature_flag_response": "control",
feature_flag_property: "control",
"$feature_flag": feature_flag.key,
},
timestamp=datetime(2023, 1, 3),
)
_create_event(
team=self.team,
event="Some other event",
distinct_id="user_test_3",
properties={feature_flag_property: "control"},
properties={
"$feature_flag_response": "control",
feature_flag_property: "control",
"$feature_flag": feature_flag.key,
},
timestamp=datetime(2023, 1, 5),
)
_create_event(
team=self.team,
event="$feature_flag_called",
distinct_id="user_test_3",
properties={feature_flag_property: "control"},
properties={
"$feature_flag_response": "control",
feature_flag_property: "control",
"$feature_flag": feature_flag.key,
},
timestamp=datetime(2023, 1, 9),
)

Expand Down Expand Up @@ -813,7 +837,12 @@ def test_query_runner_with_data_warehouse_series_no_end_date_and_nested_id(self)
team=self.team,
event="$feature_flag_called",
distinct_id=f"distinct_{variant}_{i}",
properties={feature_flag_property: variant, "$user_id": f"user_{variant}_{i}"},
properties={
"$feature_flag_response": variant,
feature_flag_property: variant,
"$feature_flag": feature_flag.key,
"$user_id": f"user_{variant}_{i}",
},
timestamp=datetime(2023, 1, i + 1),
)

Expand All @@ -827,21 +856,36 @@ def test_query_runner_with_data_warehouse_series_no_end_date_and_nested_id(self)
team=self.team,
event="$feature_flag_called",
distinct_id="distinct_test_3",
properties={feature_flag_property: "control", "$user_id": "user_test_3"},
properties={
"$feature_flag_response": "control",
feature_flag_property: "control",
"$feature_flag": feature_flag.key,
"$user_id": "user_test_3",
},
timestamp=datetime(2023, 1, 3),
)
_create_event(
team=self.team,
event="Some other event",
distinct_id="distinct_test_3",
properties={feature_flag_property: "control", "$user_id": "user_test_3"},
properties={
"$feature_flag_response": "control",
feature_flag_property: "control",
"$feature_flag": feature_flag.key,
"$user_id": "user_test_3",
},
timestamp=datetime(2023, 1, 5),
)
_create_event(
team=self.team,
event="$feature_flag_called",
distinct_id="distinct_test_3",
properties={feature_flag_property: "control", "$user_id": "user_test_3"},
properties={
"$feature_flag_response": "control",
feature_flag_property: "control",
"$feature_flag": feature_flag.key,
"$user_id": "user_test_3",
},
timestamp=datetime(2023, 1, 9),
)

Expand Down Expand Up @@ -934,7 +978,11 @@ def test_query_runner_with_data_warehouse_series_expected_query(self):
team=self.team,
event="$feature_flag_called",
distinct_id=f"user_{variant}_{i}",
properties={feature_flag_property: variant},
properties={
"$feature_flag_response": variant,
feature_flag_property: variant,
"$feature_flag": feature_flag.key,
},
timestamp=datetime(2023, 1, i + 1),
)

Expand Down Expand Up @@ -1051,7 +1099,11 @@ def test_query_runner_with_avg_math(self):
team=self.team,
event="$feature_flag_called",
distinct_id=f"user_{variant}_{i}",
properties={feature_flag_property: variant},
properties={
"$feature_flag_response": variant,
feature_flag_property: variant,
"$feature_flag": feature_flag.key,
},
timestamp=datetime(2020, 1, i + 1),
)

Expand Down Expand Up @@ -1146,27 +1198,51 @@ def test_query_runner_standard_flow(self):
{
"event": "$feature_flag_called",
"timestamp": "2020-01-02",
"properties": {ff_property: "control"},
"properties": {
"$feature_flag_response": "control",
ff_property: "control",
"$feature_flag": feature_flag.key,
},
},
],
"user_control_2": [
{"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "control"}},
{
"event": "$feature_flag_called",
"timestamp": "2020-01-02",
"properties": {ff_property: "control"},
"properties": {
"$feature_flag_response": "control",
ff_property: "control",
"$feature_flag": feature_flag.key,
},
},
],
"user_test_1": [
{"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}},
{"event": "$pageview", "timestamp": "2020-01-03", "properties": {ff_property: "test"}},
{"event": "$pageview", "timestamp": "2020-01-04", "properties": {ff_property: "test"}},
{"event": "$feature_flag_called", "timestamp": "2020-01-02", "properties": {ff_property: "test"}},
{
"event": "$feature_flag_called",
"timestamp": "2020-01-02",
"properties": {
"$feature_flag_response": "test",
ff_property: "test",
"$feature_flag": feature_flag.key,
},
},
],
"user_test_2": [
{"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}},
{"event": "$pageview", "timestamp": "2020-01-03", "properties": {ff_property: "test"}},
{"event": "$feature_flag_called", "timestamp": "2020-01-02", "properties": {ff_property: "test"}},
{
"event": "$feature_flag_called",
"timestamp": "2020-01-02",
"properties": {
"$feature_flag_response": "test",
ff_property: "test",
"$feature_flag": feature_flag.key,
},
},
],
},
self.team,
Expand Down Expand Up @@ -1245,27 +1321,51 @@ def test_query_runner_standard_flow_v2_stats(self):
{
"event": "$feature_flag_called",
"timestamp": "2020-01-02",
"properties": {ff_property: "control"},
"properties": {
"$feature_flag_response": "control",
ff_property: "control",
"$feature_flag": feature_flag.key,
},
},
],
"user_control_2": [
{"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "control"}},
{
"event": "$feature_flag_called",
"timestamp": "2020-01-02",
"properties": {ff_property: "control"},
"properties": {
"$feature_flag_response": "control",
ff_property: "control",
"$feature_flag": feature_flag.key,
},
},
],
"user_test_1": [
{"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}},
{"event": "$pageview", "timestamp": "2020-01-03", "properties": {ff_property: "test"}},
{"event": "$pageview", "timestamp": "2020-01-04", "properties": {ff_property: "test"}},
{"event": "$feature_flag_called", "timestamp": "2020-01-02", "properties": {ff_property: "test"}},
{
"event": "$feature_flag_called",
"timestamp": "2020-01-02",
"properties": {
"$feature_flag_response": "test",
ff_property: "test",
"$feature_flag": feature_flag.key,
},
},
],
"user_test_2": [
{"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}},
{"event": "$pageview", "timestamp": "2020-01-03", "properties": {ff_property: "test"}},
{"event": "$feature_flag_called", "timestamp": "2020-01-02", "properties": {ff_property: "test"}},
{
"event": "$feature_flag_called",
"timestamp": "2020-01-02",
"properties": {
"$feature_flag_response": "test",
ff_property: "test",
"$feature_flag": feature_flag.key,
},
},
],
},
self.team,
Expand Down

0 comments on commit 37c6b41

Please sign in to comment.