Secrets in envvars

Dockerfile

@@ -8,6 +8,14 @@ COPY . /code/
 WORKDIR /code
 # END gv-base
 
+# BEGIN gv-local
+FROM gv-base as gv-local
+# install dev and non-dev dependencies:
+RUN pip3 install --no-cache-dir -r requirements/dev-requirements.txt
+# Start the Django development server
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8004"]
+# END gv-local
+
 # BEGIN gv-deploy
 FROM gv-base as gv-deploy
 # install non-dev and release-only dependencies:
@@ -16,11 +24,3 @@ RUN pip3 install --no-cache-dir -r requirements/release.txt
 RUN python manage.py collectstatic --noinput
 # note: no CMD in gv-deploy -- depends on deployment
 # END gv-deploy
-
-# BEGIN gv-local
-FROM gv-base as gv-local
-# install dev and non-dev dependencies:
-RUN pip3 install --no-cache-dir -r requirements/dev-requirements.txt
-# Start the Django development server
-CMD ["python", "manage.py", "runserver", "0.0.0.0:8004"]
-# END gv-local

addon_service/authorized_storage_account/models.py

@@ -163,7 +163,10 @@ def auth_url(self) -> str | None:
         Returns None if the ExternalStorageService does not support OAuth2
         or if the initial credentials exchange has already ocurred.
         """
-        if self.credentials_format is not CredentialsFormats.OAUTH2:
+        if (
+            self.credentials_format is not CredentialsFormats.OAUTH2
+            or self.oauth2_token_metadata_id is None
+        ):
             return None
 
         state_token = self.oauth2_token_metadata.state_token
@@ -189,6 +192,10 @@ def api_base_url(self, value):
     def imp_cls(self) -> type[AddonImp]:
         return self.external_service.addon_imp.imp_cls
 
+    @property
+    def credentials_available(self) -> bool:
+        return self._credentials is not None
+
     @transaction.atomic
     def initiate_oauth2_flow(self, authorized_scopes=None):
         if self.credentials_format is not CredentialsFormats.OAUTH2:

addon_service/authorized_storage_account/serializers.py

@@ -66,6 +66,7 @@ def __init__(self, *args, **kwargs):
         related_link_view_name=view_names.related_view(RESOURCE_TYPE),
     )
     credentials = CredentialsField(write_only=True, required=False)
+    initiate_oauth = serializers.BooleanField(write_only=True, required=False)
 
     included_serializers = {
         "account_owner": "addon_service.serializers.UserReferenceSerializer",
@@ -91,12 +92,16 @@ def create(self, validated_data):
         except ModelValidationError as e:
             raise serializers.ValidationError(e)
 
-        if external_service.credentials_format is CredentialsFormats.OAUTH2:
+        if (
+            validated_data.get("initiate_oauth", False)
+            and external_service.credentials_format is CredentialsFormats.OAUTH2
+        ):
             authorized_account.initiate_oauth2_flow(
                 validated_data.get("authorized_scopes")
             )
-        else:
+        elif validated_data.get("credentials"):
             authorized_account.credentials = validated_data["credentials"]
+
         try:
             authorized_account.save()
         except ModelValidationError as e:
@@ -120,4 +125,6 @@ class Meta:
             "credentials",
             "default_root_folder",
             "external_storage_service",
+            "initiate_oauth",
+            "credentials_available",
         ]

addon_service/credentials/encryption.py

@@ -132,6 +132,12 @@ def _derive_multifernet_key(
     key_params: KeyParameters,
     /,  # positional-only params for cache-friendliness
 ) -> fernet.MultiFernet:
+    if not settings.GRAVYVALET_ENCRYPT_SECRET:
+        raise RuntimeError(
+            "gravyvalet can not keep your secrets without a GRAVYVALET_ENCRYPT_SECRET"
+            " -- ideally chosen by strong randomness, with around 256 bits of entropy"
+            " (e.g. 64 hex digits; 60 d20 rolls; 20 words of a 10000-word vocabulary)"
+        )
     # https://cryptography.io/en/latest/fernet/#cryptography.fernet.MultiFernet
     return fernet.MultiFernet(
         [

addon_service/management/commands/rotate_encryption.py

@@ -0,0 +1,9 @@
+from django.core.management.base import BaseCommand
+
+from addon_service.tasks.key_rotation import schedule_encryption_rotation__celery
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **kwargs):
+        _task = schedule_encryption_rotation__celery.apply_async()
+        self.stdout.write(self.style.SUCCESS(f"scheduled task {_task}"))

addon_service/oauth/models.py

@@ -1,6 +1,7 @@
 from datetime import timedelta
 
 from asgiref.sync import sync_to_async
+from django.conf import settings
 from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ValidationError
 from django.db import (
@@ -28,7 +29,6 @@ class OAuth2ClientConfig(AddonsServiceBaseModel):
     token_endpoint_url = models.URLField(null=False)
     # The registered ID of the OAuth client
     client_id = models.CharField(null=True)
-    client_secret = models.CharField(null=True)
 
     class Meta:
         verbose_name = "OAuth2 Client Config"
@@ -38,6 +38,10 @@ class Meta:
     def __repr__(self):
         return f'<{self.__class__.__qualname__}(pk="{self.pk}", auth_uri="{self.auth_uri}", client_id="{self.client_id}")>'
 
+    @property
+    def client_secret(self):
+        return settings.OAUTH_SECRETS.get(self.client_id)
+
     __str__ = __repr__
 
 

addon_service/tests/_factories.py

@@ -33,7 +33,6 @@ class Meta:
     auth_callback_url = "https://osf.example/auth/callback"
     token_endpoint_url = "https://api.example.com/oauth/token"
     client_id = factory.Faker("word")
-    client_secret = factory.Faker("word")
 
 
 class AddonOperationInvocationFactory(DjangoModelFactory):

addon_service/tests/_helpers.py

@@ -187,13 +187,16 @@ def get_test_request(user=None, method="get", path="", cookies=None):
     return _request
 
 
+@contextlib.contextmanager
 def patch_encryption_key_derivation():
-    # expect to call scrypt with all the following params:
-    def _mock_scrypt(secret, salt, n, r, p, dklen, maxmem):
-        # some random derived key
-        return b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb"
+    _fake_secret = b"this is fine"
+    _some_random_key = b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb"
 
-    return patch(
+    with patch(
+        "addon_service.credentials.encryption.settings.GRAVYVALET_ENCRYPT_SECRET",
+        _fake_secret,
+    ), patch(
         "addon_service.credentials.encryption.hashlib.scrypt",
-        side_effect=_mock_scrypt,
-    )
+        return_value=_some_random_key,
+    ):
+        yield

addon_service/tests/e2e_tests/test_oauth_flow.py

@@ -26,6 +26,7 @@ def _make_post_payload(*, external_service, capabilities=None, credentials=None)
             "type": "authorized-storage-accounts",
             "attributes": {
                 "authorized_capabilities": [AddonCapabilities.ACCESS.name],
+                "initiate_oauth": True,
             },
             "relationships": {
                 "external_storage_service": {

addon_service/tests/test_by_type/test_authorized_storage_account.py

@@ -56,6 +56,7 @@ def _make_post_payload(
             "attributes": {
                 "authorized_capabilities": capabilities,
                 "api_base_url": api_root,
+                "initiate_oauth": True,
             },
             "relationships": {
                 "external_storage_service": {

app/celery.py

@@ -68,4 +68,4 @@ def _enqueue_handler_task(body, message):
         ]
 
 
-app.steps["consumer"].add(OsfBackchannelConsumerStep)
+# app.steps["consumer"].add(OsfBackchannelConsumerStep)

app/env.py

@@ -4,6 +4,17 @@
 import os
 
 
+DEBUG = bool(os.environ.get("DEBUG"))  # any non-empty value enables debug mode
+SECRET_KEY = os.environ.get("SECRET_KEY")  # used by django for cryptographic signing
+ALLOWED_HOSTS = list(filter(bool, os.environ.get("ALLOWED_HOSTS", "").split(",")))
+CORS_ALLOWED_ORIGINS = tuple(
+    filter(bool, os.environ.get("CORS_ALLOWED_ORIGINS", "").split(","))
+)
+SENTRY_DSN = os.environ.get("SENTRY_DSN")
+
+###
+# databases
+
 POSTGRES_DB = os.environ.get("POSTGRES_DB")
 POSTGRES_USER = os.environ.get("POSTGRES_USER")
 POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD")
@@ -17,17 +28,20 @@
 OSFDB_PORT = os.environ.get("OSFDB_PORT", "5432")
 OSFDB_CONN_MAX_AGE = os.environ.get("OSFDB_CONN_MAX_AGE", 0)
 OSFDB_SSLMODE = os.environ.get("OSFDB_SSLMODE", "prefer")
+
+###
+# for interacting with osf
+
 OSF_SENSITIVE_DATA_SECRET = os.environ.get("OSF_SENSITIVE_DATA_SECRET", "")
 OSF_SENSITIVE_DATA_SALT = os.environ.get("OSF_SENSITIVE_DATA_SALT", "")
-
-SECRET_KEY = os.environ.get("SECRET_KEY")
 OSF_HMAC_KEY = os.environ.get("OSF_HMAC_KEY")
 OSF_HMAC_EXPIRATION_SECONDS = int(os.environ.get("OSF_HMAC_EXPIRATION_SECONDS", 110))
-
 OSF_BASE_URL = os.environ.get("OSF_BASE_URL", "https://osf.example")
 OSF_API_BASE_URL = os.environ.get("OSF_API_BASE_URL", "https://api.osf.example")
 
+###
 # amqp/celery
+
 AMQP_BROKER_URL = os.environ.get(
     "AMQP_BROKER_URL", "amqp://guest:[email protected]:5672"
 )
@@ -36,15 +50,6 @@
     "OSF_BACKCHANNEL_QUEUE_NAME", "account_status_changes"
 )
 
-# any non-empty value enables debug mode:
-DEBUG = bool(os.environ.get("DEBUG"))
-
-# comma-separated list:
-ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(",")
-CORS_ALLOWED_ORIGINS = tuple(
-    filter(bool, os.environ.get("CORS_ALLOWED_ORIGINS", "").split(","))
-)
-
 ###
 # credentials encryption secrets and parameters
 #
@@ -56,7 +61,7 @@
 # 2. call `.rotate_encryption()` on every `ExternalCredentials` (perhaps via
 #    celery tasks in `addon_service.tasks.key_rotation`)
 # 3. remove the old secret from GRAVYVALET_ENCRYPT_SECRET_PRIORS
-GRAVYVALET_ENCRYPT_SECRET = os.environ.get("GRAVYVALET_ENCRYPT_SECRET")
+GRAVYVALET_ENCRYPT_SECRET: str | None = os.environ.get("GRAVYVALET_ENCRYPT_SECRET")
 GRAVYVALET_ENCRYPT_SECRET_PRIORS = tuple(
     filter(bool, os.environ.get("GRAVYVALET_ENCRYPT_SECRET_PRIORS", "").split(","))
 )
@@ -74,3 +79,11 @@
 )
 # END credentials encryption secrets and parameters
 ###
+
+
+###
+# OAuth Client Settings
+BOX_OAUTH2_CLIENT_ID: str | None = os.environ.get("BOX_CLIENT_ID", "box")
+BOX_OAUTH2_CLIENT_SECRET: str | None = os.environ.get("BOX_SECRET", "box_secret")
+# END OAuth Client Settings
+###

app/settings.py

@@ -1,20 +1,30 @@
+import logging
 from pathlib import Path
 
 from app import env
 
 
+_logger = logging.getLogger(__name__)
+
+
+if env.SENTRY_DSN:
+    try:
+        import sentry_sdk
+    except ImportError:
+        _logger.warning("SENTRY_DSN defined but sentry_sdk not installed!")
+    else:
+        sentry_sdk.init(
+            dsn=env.SENTRY_DSN,
+            environment=env.OSF_BASE_URL,
+        )
+
+
 SECRET_KEY = env.SECRET_KEY
 OSF_HMAC_KEY = env.OSF_HMAC_KEY or "lmaoooooo"
 OSF_HMAC_EXPIRATION_SECONDS = env.OSF_HMAC_EXPIRATION_SECONDS
 
-if not env.DEBUG and not env.GRAVYVALET_ENCRYPT_SECRET:
-    raise RuntimeError(
-        "pls set `GRAVYVALET_ENCRYPT_SECRET` environment variable to something safely random"
-    )
-GRAVYVALET_ENCRYPT_SECRET: bytes = (
-    env.GRAVYVALET_ENCRYPT_SECRET.encode()
-    if env.GRAVYVALET_ENCRYPT_SECRET
-    else b"this is fine"
+GRAVYVALET_ENCRYPT_SECRET: bytes | None = (
+    env.GRAVYVALET_ENCRYPT_SECRET.encode() if env.GRAVYVALET_ENCRYPT_SECRET else None
 )
 GRAVYVALET_ENCRYPT_SECRET_PRIORS: tuple[bytes, ...] = tuple(
     _prior.encode() for _prior in env.GRAVYVALET_ENCRYPT_SECRET_PRIORS
@@ -192,6 +202,7 @@
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.1/howto/static-files/
 
+STATIC_ROOT = BASE_DIR / "static"
 STATIC_URL = "/static/"
 
 OSF_SENSITIVE_DATA_SECRET = env.OSF_SENSITIVE_DATA_SECRET
@@ -204,3 +215,9 @@
 AMQP_BROKER_URL = env.AMQP_BROKER_URL
 OSF_BACKCHANNEL_QUEUE_NAME = env.OSF_BACKCHANNEL_QUEUE_NAME
 GV_QUEUE_NAME_PREFIX = env.GV_QUEUE_NAME_PREFIX
+
+
+###
+# Mapping from OAuth Client IDs to Secrets
+# DB should store the Client IDs to serve as a shared lookup, but secrets should live in ENVVARS
+OAUTH_SECRETS = {env.BOX_OAUTH2_CLIENT_ID: env.BOX_OAUTH2_CLIENT_SECRET}

docker-compose.yml

@@ -3,10 +3,12 @@ version: "3.8"
 services:
 
   gravyvalet:
-    build: .
+    build:
+      context: .
+      target: gv-local
     restart: unless-stopped
     command: python manage.py runserver 0.0.0.0:8004
-    environment:
+    environment: &gv_environment
       DEBUG: 1
       DJANGO_SETTINGS_MODULE: app.settings
       PYTHONUNBUFFERED: 1
@@ -20,6 +22,7 @@ services:
       OSF_BASE_URL: "http://192.168.168.167:5000"
       OSF_API_BASE_URL: "http://192.168.168.167:8000"
       AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672"
+      GRAVYVALET_ENCRYPT_SECRET: "this is fine"
     ports:
       - 8004:8004
     stdin_open: true
@@ -30,24 +33,17 @@ services:
       - postgres
 
   celeryworker:
-    build: .
+    build:
+      context: .
+      target: gv-local
     restart: unless-stopped
     command: /usr/local/bin/celery --app app worker --uid daemon -l INFO
     depends_on:
       - rabbitmq
       - postgres
     volumes:
       - ./:/code:cached
-    environment:
-      DEBUG: 1
-      DJANGO_SETTINGS_MODULE: app.settings
-      POSTGRES_HOST: postgres
-      POSTGRES_DB: gravyvalet
-      POSTGRES_USER: postgres
-      SECRET_KEY: so-secret
-      OSF_BASE_URL: "http://192.168.168.167:5000"
-      OSF_API_BASE_URL: "http://192.168.168.167:8000"
-      AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672"
+    environment: *gv_environment
     stdin_open: true
 
   postgres:

requirements/dev-requirements.txt

@@ -10,7 +10,3 @@ flake8
 black==24.*
 isort
 pre-commit
-
-# ASGI server for local use
-# https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/daphne/#integration-with-runserver
-daphne

requirements/release.txt

@@ -1,3 +1,4 @@
 -r ./requirements.txt
 
 # Requirements to be installed on server deployments
+sentry-sdk==2.7.1

requirements/requirements.txt

@@ -1,4 +1,5 @@
 Django==4.2.7
+daphne==4.1.2
 psycopg>=3.1.8
 djangorestframework==3.14.0
 djangorestframework-jsonapi==6.1.0