DM-48476: Sentry - the most basic integration possible

Let's use Sentry in the most basic possible way and see what happens.

docs/user-guide/environment-variables.rst

@@ -30,6 +30,11 @@ See the `Phalanx documentation for Times Square <https://phalanx.lsst.io/applica
    (string) The base URL of the Rubin Science Platform environment.
    This is used for creating URLs to services, such as JupyterHub.
 
+.. envvar:: TS_ENVIRONMENT_NAME
+
+   (string) The name of the Rubin Science Platform environment.
+   This is used as the Sentry environment.
+
 .. envvar:: TS_GAFAELFAWR_TOKEN
 
    (secret string) This token is used to make an admin API call to Gafaelfawr to get a token for the user.
@@ -74,3 +79,8 @@ See the `Phalanx documentation for Times Square <https://phalanx.lsst.io/applica
 .. envvar:: TS_GITHUB_ORGS
 
    (string) A comma-separated list of GitHub organizations that Times Square will sync notebooks from. This is used to filter out incidental GitHub App installations from the general public.
+
+.. envvar:: TS_SENTRY_TRACES_SAMPLE_RATE
+
+   (float) The percentage of transactions to send to Sentry, expressed as a float between 0 and 1. 0 means send no traces, 1 means send every trace.
+

pyproject.toml

@@ -120,6 +120,9 @@ extend-exclude = [
 "src/timessquare/config.py" = [
     "FBT001",  # Pydantic validators take boolean arguments
 ]
+"src/timessquare/sentry.py" = [
+    "S311",  # We're not using random.random for cryptography
+]
 "tests/**" = [
     "C901",    # tests are allowed to be complex, sometimes that's convenient
     "D101",    # tests don't need docstrings

requirements/dev.txt

@@ -1584,6 +1584,7 @@ urllib3==2.3.0 \
     --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \
     --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d
     # via
+    #   -c requirements/main.txt
     #   documenteer
     #   requests
     #   sphinx-prompt

requirements/main.in

@@ -27,4 +27,5 @@ markdown-it-py[linkify,plugins]
 mdformat
 mdformat-gfm
 PyYAML
+sentry-sdk
 sse-starlette

requirements/main.txt

@@ -140,6 +140,7 @@ certifi==2024.12.14 \
     # via
     #   httpcore
     #   httpx
+    #   sentry-sdk
 cffi==1.17.1 ; implementation_name == 'pypy' or platform_python_implementation != 'PyPy' \
     --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \
     --hash=sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2 \
@@ -1238,6 +1239,10 @@ safir-logging==9.1.1 \
     --hash=sha256:5b268259f282502471fdb6a2087d2e5a010fa88d73cfa71a32fc5c656562830a \
     --hash=sha256:e48e905ca30cb7f2ee95d54b22fd0678511b18b59275203a611368fc0da4ea0c
     # via safir
+sentry-sdk==2.20.0 \
+    --hash=sha256:afa82713a92facf847df3c6f63cec71eb488d826a50965def3d7722aa6f0fdab \
+    --hash=sha256:c359a1edf950eb5e80cffd7d9111f3dbeef57994cb4415df37d39fda2cf22364
+    # via -r requirements/main.in
 six==1.17.0 \
     --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
     --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
@@ -1379,6 +1384,10 @@ uritemplate==4.1.1 \
     --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \
     --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e
     # via gidgethub
+urllib3==2.3.0 \
+    --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \
+    --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d
+    # via sentry-sdk
 uvicorn==0.34.0 \
     --hash=sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4 \
     --hash=sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9

requirements/tox.txt

@@ -218,6 +218,7 @@ urllib3==2.3.0 \
     --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d
     # via
     #   -c requirements/dev.txt
+    #   -c requirements/main.txt
     #   docker
     #   requests
 uv==0.5.20 \

src/timessquare/config.py

@@ -64,6 +64,16 @@ class Config(BaseSettings):
         ),
     ]
 
+    environment_name: Annotated[
+        str,
+        Field(
+            alias="TS_ENVIRONMENT_NAME",
+            description=(
+                "The Phalanx name of the Rubin Science Platform environment."
+            ),
+        ),
+    ]
+
     gafaelfawr_token: Annotated[
         SecretStr,
         Field(
@@ -226,6 +236,28 @@ class Config(BaseSettings):
         ),
     ] = None
 
+    sentry_dsn: Annotated[
+        str | None,
+        Field(
+            alias="TS_SENTRY_DSN",
+            description="DSN for sending events to Sentry.",
+        ),
+    ] = None
+
+    sentry_traces_sample_rate: Annotated[
+        float,
+        Field(
+            alias="TS_SENTRY_TRACES_SAMPLE_RATE",
+            description=(
+                "The percentage of transactions to send to Sentry, expressed "
+                "as a float between 0 and 1. 0 means send no traces, 1 means "
+                "send every trace."
+            ),
+            ge=0,
+            le=1,
+        ),
+    ] = 0
+
     @field_validator("path_prefix")
     @classmethod
     def validate_path_prefix(cls, v: str) -> str:

src/timessquare/main.py

@@ -15,6 +15,7 @@
 from importlib.metadata import version
 from pathlib import Path
 
+import sentry_sdk
 from fastapi import FastAPI
 from fastapi.openapi.utils import get_openapi
 from safir.database import create_database_engine, is_database_current
@@ -32,9 +33,19 @@
 from .handlers.external import external_router
 from .handlers.internal import internal_router
 from .handlers.v1 import v1_router
+from .sentry import make_traces_sampler
 
 __all__ = ["app", "config"]
 
+sentry_sdk.init(
+    dsn=config.sentry_dsn,
+    environment=config.environment_name,
+    traces_sample_rate=config.sentry_traces_sample_rate,
+    traces_sampler=make_traces_sampler(
+        sample_rate=config.sentry_traces_sample_rate
+    ),
+)
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncIterator:

src/timessquare/sentry.py

@@ -0,0 +1,35 @@
+"""Sentry integration helpers."""
+
+import random
+from collections.abc import Callable
+from typing import Any
+
+
+def make_traces_sampler(
+    sample_rate: float,
+) -> Callable[[dict[str, Any]], float | bool]:
+    """Don't sample probes, sample other requests at the configured rate.
+
+    Sentry can be configured either with ``traces_sample_rate`` or
+    traces_sampler, but not both, so we have to re-implement the sampling rate
+    logic.
+
+    https://github.com/getsentry/sentry/issues/78457
+    """
+
+    def filter_transactions(context: dict[str, Any]) -> float | bool:
+        try:
+            headers = context["asgi_scope"]["headers"]
+            user_agent = (
+                header[1]
+                for header in headers
+                if header[0].lower() == "user-agent"
+            ).__next__()
+            if user_agent.startswith("kube-probe"):
+                return False
+        except (IndexError, StopIteration):
+            pass
+
+        return random.random() < float(sample_rate)
+
+    return filter_transactions

src/timessquare/worker/main.py

@@ -8,6 +8,7 @@
 
 import arq
 import httpx
+import sentry_sdk
 import structlog
 from safir.database import create_database_engine, is_database_current
 from safir.dependencies.db_session import db_session_dependency
@@ -31,6 +32,12 @@
     repo_removed,
 )
 
+sentry_sdk.init(
+    dsn=config.sentry_dsn,
+    traces_sample_rate=config.sentry_traces_sample_rate,
+    environment=config.environment_name,
+)
+
 
 async def startup(ctx: dict[Any, Any]) -> None:
     """Set up the worker context."""

tox.ini

@@ -47,6 +47,7 @@ setenv =
     SAFIR_PROFILE = development
     TS_ALEMBIC_CONFIG_PATH = {toxinidir}/alembic.ini
     TS_ENVIRONMENT_URL = https://test.example.com
+    TS_ENVIRONMENT_NAME = testing
     TS_DATABASE_URL = postgresql://timessquare@localhost:5433/timessquare
     TS_DATABASE_PASSWORD = INSECURE-PASSWORD
     TS_REDIS_URL = redis://localhost:6379/0
@@ -86,6 +87,7 @@ commands = pre-commit run --all-files
 description = Run Alembic against a test database
 setenv =
     TS_ENVIRONMENT_URL = https://test.example.com
+    TS_ENVIRONMENT_NAME = testing
     TS_PATH_PREFIX = /times-square/api
     TS_ALEMBIC_CONFIG_PATH = {toxinidir}/alembic.ini
     TS_DATABASE_URL = postgresql://[email protected]:5432/timessquare
@@ -107,6 +109,7 @@ commands =
     times-square {posargs}
 setenv =
     TS_ENVIRONMENT_URL = https://test.example.com
+    TS_ENVIRONMENT_NAME = testing
     TS_PATH_PREFIX = /times-square/api
     TS_ALEMBIC_CONFIG_PATH = {toxinidir}/alembic.ini
     TS_DATABASE_URL = postgresql://[email protected]:5432/timessquare
@@ -126,6 +129,7 @@ whitelist_externals =
 setenv =
     SAFIR_PROFILE = development
     TS_ENVIRONMENT_URL = https://test.example.com
+    TS_ENVIRONMENT_NAME = testing
     TS_PATH_PREFIX = /times-square/api
     TS_ALEMBIC_CONFIG_PATH = {toxinidir}/alembic.ini
     TS_DATABASE_URL = postgresql://[email protected]:5432/timessquare
@@ -148,6 +152,7 @@ commands_pre =
 description = Build documentation (HTML) with Sphinx.
 setenv =
     TS_ENVIRONMENT_URL = https://test.example.com
+    TS_ENVIRONMENT_NAME = testing
     TS_DATABASE_URL = postgresql://timessquare@localhost:5433/timessquare
     TS_DATABASE_PASSWORD = INSECURE-PASSWORD
     TS_GAFAELFAWR_TOKEN = gt-eOfLolxU8FJ1xr08U7RTbg.Jr-KHSeISXwR5GXHiLemhw
@@ -165,6 +170,7 @@ commands =
 description = Check links in documentation.
 setenv =
     TS_ENVIRONMENT_URL = https://test.example.com
+    TS_ENVIRONMENT_NAME = testing
     TS_DATABASE_URL = postgresql://timessquare@localhost:5433/timessquare
     TS_DATABASE_PASSWORD = INSECURE-PASSWORD
     TS_GAFAELFAWR_TOKEN = gt-eOfLolxU8FJ1xr08U7RTbg.Jr-KHSeISXwR5GXHiLemhw