feat(flags): Add integration for custom tracking of flag evaluations

sentry_sdk/integrations/featureflags.py

@@ -0,0 +1,44 @@
+from sentry_sdk.flag_utils import flag_error_processor
+
+import sentry_sdk
+from sentry_sdk.integrations import Integration
+
+
+class FeatureFlagsIntegration(Integration):
+    """
+    Sentry integration for buffering feature flags manually with an API and capturing them on
+    error events. We recommend you do this on each flag evaluation. Flags are buffered per Sentry
+    scope.
+
+    See the [feature flag documentation](https://develop.sentry.dev/sdk/expected-features/#feature-flags)
+    for more information.
+
+    @example
+    ```
+    import sentry_sdk
+    from sentry_sdk.integrations.featureflags import FeatureFlagsIntegration, add_feature_flag
+
+    sentry_sdk.init(dsn="my_dsn", integrations=[FeatureFlagsIntegration()]);
+
+    add_feature_flag('my-flag', true);
+    sentry_sdk.capture_exception(Exception('broke')); // 'my-flag' should be captured on this Sentry event.
+    ```
+    """
+
+    identifier = "featureflags"
+
+    @staticmethod
+    def setup_once():
+        # type: () -> None
+        scope = sentry_sdk.get_current_scope()
+        scope.add_error_processor(flag_error_processor)
+
+
+def add_feature_flag(flag, result):
+    # type: (str, bool) -> None
+    """
+    Records a flag and its value to be sent on subsequent error events. We recommend you do this
+    on flag evaluations. Flags are buffered per Sentry scope and limited to 100 per event.
+    """
+    flags = sentry_sdk.get_current_scope().flags
+    flags.set(flag, result)

tests/integrations/featureflags/test_featureflags.py

@@ -0,0 +1,142 @@
+import asyncio
+import concurrent.futures as cf
+
+import pytest
+
+import sentry_sdk
+from sentry_sdk.integrations import _processed_integrations, _installed_integrations
+from sentry_sdk.integrations.featureflags import (
+    FeatureFlagsIntegration,
+    add_feature_flag,
+)
+
+
+@pytest.fixture
+def uninstall_integration():
+    """Forces the next call to sentry_init to re-install/setup an integration."""
+
+    def inner(identifier):
+        _processed_integrations.discard(identifier)
+        _installed_integrations.discard(identifier)
+
+    return inner
+
+
+def test_featureflags_integration(sentry_init, capture_events, uninstall_integration):
+    uninstall_integration(FeatureFlagsIntegration.identifier)
+    sentry_init(integrations=[FeatureFlagsIntegration()])
+
+    add_feature_flag("hello", False)
+    add_feature_flag("world", True)
+    add_feature_flag("other", False)
+
+    events = capture_events()
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 1
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "world", "result": True},
+            {"flag": "other", "result": False},
+        ]
+    }
+
+
+def test_featureflags_integration_threaded(
+    sentry_init, capture_events, uninstall_integration
+):
+    uninstall_integration(FeatureFlagsIntegration.identifier)
+    sentry_init(integrations=[FeatureFlagsIntegration()])
+    events = capture_events()
+
+    # Capture an eval before we split isolation scopes.
+    add_feature_flag("hello", False)
+
+    def task(flag_key):
+        # Creates a new isolation scope for the thread.
+        # This means the evaluations in each task are captured separately.
+        with sentry_sdk.isolation_scope():
+            add_feature_flag(flag_key, False)
+            # use a tag to identify to identify events later on
+            sentry_sdk.set_tag("task_id", flag_key)
+            sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    # Run tasks in separate threads
+    with cf.ThreadPoolExecutor(max_workers=2) as pool:
+        pool.map(task, ["world", "other"])
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "other", "result": False},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "world", "result": False},
+        ]
+    }
+
+
+def test_featureflags_integration_asyncio(
+    sentry_init, capture_events, uninstall_integration
+):
+    uninstall_integration(FeatureFlagsIntegration.identifier)
+    sentry_init(integrations=[FeatureFlagsIntegration()])
+    events = capture_events()
+
+    # Capture an eval before we split isolation scopes.
+    add_feature_flag("hello", False)
+
+    async def task(flag_key):
+        # Creates a new isolation scope for the thread.
+        # This means the evaluations in each task are captured separately.
+        with sentry_sdk.isolation_scope():
+            add_feature_flag(flag_key, False)
+            # use a tag to identify to identify events later on
+            sentry_sdk.set_tag("task_id", flag_key)
+            sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    async def runner():
+        return asyncio.gather(task("world"), task("other"))
+
+    asyncio.run(runner())
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "other", "result": False},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "world", "result": False},
+        ]
+    }

tests/integrations/launchdarkly/test_launchdarkly.py

@@ -57,7 +57,7 @@ def task(flag_key):
         # This means the evaluations in each task are captured separately.
         with sentry_sdk.isolation_scope():
             client.variation(flag_key, context, False)
-            return [f["flag"] for f in sentry_sdk.get_current_scope().flags.get()]
+            return sentry_sdk.get_current_scope().flags.get()
 
     td.update(td.flag("hello").variation_for_all(True))
     td.update(td.flag("world").variation_for_all(False))
@@ -67,8 +67,14 @@ def task(flag_key):
     with cf.ThreadPoolExecutor(max_workers=2) as pool:
         results = list(pool.map(task, ["world", "other"]))
 
-    assert results[0] == ["hello", "world"]
-    assert results[1] == ["hello", "other"]
+    assert results[0] == [
+        {"flag": "hello", "result": True},
+        {"flag": "world", "result": False},
+    ]
+    assert results[1] == [
+        {"flag": "hello", "result": True},
+        {"flag": "other", "result": False},
+    ]
 
 
 def test_launchdarkly_integration_asyncio(sentry_init):
@@ -81,7 +87,7 @@ def test_launchdarkly_integration_asyncio(sentry_init):
     async def task(flag_key):
         with sentry_sdk.isolation_scope():
             client.variation(flag_key, context, False)
-            return [f["flag"] for f in sentry_sdk.get_current_scope().flags.get()]
+            return sentry_sdk.get_current_scope().flags.get()
 
     async def runner():
         return asyncio.gather(task("world"), task("other"))
@@ -91,8 +97,14 @@ async def runner():
     client.variation("hello", context, False)
 
     results = asyncio.run(runner()).result()
-    assert results[0] == ["hello", "world"]
-    assert results[1] == ["hello", "other"]
+    assert results[0] == [
+        {"flag": "hello", "result": True},
+        {"flag": "world", "result": False},
+    ]
+    assert results[1] == [
+        {"flag": "hello", "result": True},
+        {"flag": "other", "result": False},
+    ]
 
 
 def test_launchdarkly_integration_did_not_enable(monkeypatch):

tests/integrations/openfeature/test_openfeature.py

@@ -44,13 +44,19 @@ def task(flag):
         # Create a new isolation scope for the thread. This means the flags
         with sentry_sdk.isolation_scope():
             client.get_boolean_value(flag, default_value=False)
-            return [f["flag"] for f in sentry_sdk.get_current_scope().flags.get()]
+            return sentry_sdk.get_current_scope().flags.get()
 
     with cf.ThreadPoolExecutor(max_workers=2) as pool:
         results = list(pool.map(task, ["world", "other"]))
 
-    assert results[0] == ["hello", "world"]
-    assert results[1] == ["hello", "other"]
+    assert results[0] == [
+        {"flag": "hello", "result": True},
+        {"flag": "world", "result": False},
+    ]
+    assert results[1] == [
+        {"flag": "hello", "result": True},
+        {"flag": "other", "result": False},
+    ]
 
 
 def test_openfeature_integration_asyncio(sentry_init):
@@ -59,7 +65,7 @@ def test_openfeature_integration_asyncio(sentry_init):
     async def task(flag):
         with sentry_sdk.isolation_scope():
             client.get_boolean_value(flag, default_value=False)
-            return [f["flag"] for f in sentry_sdk.get_current_scope().flags.get()]
+            return sentry_sdk.get_current_scope().flags.get()
 
     async def runner():
         return asyncio.gather(task("world"), task("other"))
@@ -76,5 +82,11 @@ async def runner():
     client.get_boolean_value("hello", default_value=False)
 
     results = asyncio.run(runner()).result()
-    assert results[0] == ["hello", "world"]
-    assert results[1] == ["hello", "other"]
+    assert results[0] == [
+        {"flag": "hello", "result": True},
+        {"flag": "world", "result": False},
+    ]
+    assert results[1] == [
+        {"flag": "hello", "result": True},
+        {"flag": "other", "result": False},
+    ]