From caa247df2af25d6addafc5462a30c675fc10d0a2 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:09:36 -0400 Subject: [PATCH 01/20] fixups for deployment - allow running management commands with neither DEBUG nor GRAVYVALET_ENCRYPT_SECRET set - stop supplying a fake GRAVYVALET_ENCRYPT_SECRET in app.settings -- instead, set a fake GRAVYVALET_ENCRYPT_SECRET in docker-compose.yml --- addon_service/credentials/encryption.py | 6 ++++++ app/env.py | 2 +- app/settings.py | 10 ++-------- docker-compose.yml | 14 +++----------- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/addon_service/credentials/encryption.py b/addon_service/credentials/encryption.py index 45090944..e597fd15 100644 --- a/addon_service/credentials/encryption.py +++ b/addon_service/credentials/encryption.py @@ -132,6 +132,12 @@ def _derive_multifernet_key( key_params: KeyParameters, /, # positional-only params for cache-friendliness ) -> fernet.MultiFernet: + if not settings.GRAVYVALET_ENCRYPT_SECRET: + raise RuntimeError( + "gravyvalet can not keep your secrets without a GRAVYVALET_ENCRYPT_SECRET" + " -- ideally chosen by strong randomness, with around 256 bits of entropy" + " (e.g. 64 hex digits; 60 d20 rolls; 20 words of a 10000-word vocabulary)" + ) # https://cryptography.io/en/latest/fernet/#cryptography.fernet.MultiFernet return fernet.MultiFernet( [ diff --git a/app/env.py b/app/env.py index 8d6edda3..98a8c9bc 100644 --- a/app/env.py +++ b/app/env.py @@ -56,7 +56,7 @@ # 2. call `.rotate_encryption()` on every `ExternalCredentials` (perhaps via # celery tasks in `addon_service.tasks.key_rotation`) # 3. remove the old secret from GRAVYVALET_ENCRYPT_SECRET_PRIORS -GRAVYVALET_ENCRYPT_SECRET = os.environ.get("GRAVYVALET_ENCRYPT_SECRET") +GRAVYVALET_ENCRYPT_SECRET: str | None = os.environ.get("GRAVYVALET_ENCRYPT_SECRET") GRAVYVALET_ENCRYPT_SECRET_PRIORS = tuple( filter(bool, os.environ.get("GRAVYVALET_ENCRYPT_SECRET_PRIORS", "").split(",")) ) diff --git a/app/settings.py b/app/settings.py index 6bc9c8d1..30443488 100644 --- a/app/settings.py +++ b/app/settings.py @@ -7,14 +7,8 @@ OSF_HMAC_KEY = env.OSF_HMAC_KEY or "lmaoooooo" OSF_HMAC_EXPIRATION_SECONDS = env.OSF_HMAC_EXPIRATION_SECONDS -if not env.DEBUG and not env.GRAVYVALET_ENCRYPT_SECRET: - raise RuntimeError( - "pls set `GRAVYVALET_ENCRYPT_SECRET` environment variable to something safely random" - ) -GRAVYVALET_ENCRYPT_SECRET: bytes = ( - env.GRAVYVALET_ENCRYPT_SECRET.encode() - if env.GRAVYVALET_ENCRYPT_SECRET - else b"this is fine" +GRAVYVALET_ENCRYPT_SECRET: bytes | None = ( + env.GRAVYVALET_ENCRYPT_SECRET.encode() if env.GRAVYVALET_ENCRYPT_SECRET else None ) GRAVYVALET_ENCRYPT_SECRET_PRIORS: tuple[bytes, ...] = tuple( _prior.encode() for _prior in env.GRAVYVALET_ENCRYPT_SECRET_PRIORS diff --git a/docker-compose.yml b/docker-compose.yml index 63719bc5..3df5ea04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: build: . restart: unless-stopped command: python manage.py runserver 0.0.0.0:8004 - environment: + environment: &gv_environment DEBUG: 1 DJANGO_SETTINGS_MODULE: app.settings PYTHONUNBUFFERED: 1 @@ -20,6 +20,7 @@ services: OSF_BASE_URL: "http://192.168.168.167:5000" OSF_API_BASE_URL: "http://192.168.168.167:8000" AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + GRAVYVALET_ENCRYPT_SECRET: "this is fine" ports: - 8004:8004 stdin_open: true @@ -38,16 +39,7 @@ services: - postgres volumes: - ./:/code:cached - environment: - DEBUG: 1 - DJANGO_SETTINGS_MODULE: app.settings - POSTGRES_HOST: postgres - POSTGRES_DB: gravyvalet - POSTGRES_USER: postgres - SECRET_KEY: so-secret - OSF_BASE_URL: "http://192.168.168.167:5000" - OSF_API_BASE_URL: "http://192.168.168.167:8000" - AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + environment: *gv_environment stdin_open: true postgres: From bab57d8d9ff142059dcd3dcf9f71a38e779cacc5 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:25:50 -0400 Subject: [PATCH 02/20] add STATIC_ROOT --- app/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/settings.py b/app/settings.py index 30443488..ec83c8ea 100644 --- a/app/settings.py +++ b/app/settings.py @@ -186,6 +186,7 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ +STATIC_ROOT = BASE_DIR / "static" STATIC_URL = "/static/" OSF_SENSITIVE_DATA_SECRET = env.OSF_SENSITIVE_DATA_SECRET From 91f274a48400eda80e73931bb054dec4a8cf9c99 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:37:50 -0400 Subject: [PATCH 03/20] allow testing without secrets --- addon_service/tests/_helpers.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/addon_service/tests/_helpers.py b/addon_service/tests/_helpers.py index 920f9830..53ba2299 100644 --- a/addon_service/tests/_helpers.py +++ b/addon_service/tests/_helpers.py @@ -187,13 +187,16 @@ def get_test_request(user=None, method="get", path="", cookies=None): return _request +@contextlib.contextmanager def patch_encryption_key_derivation(): - # expect to call scrypt with all the following params: - def _mock_scrypt(secret, salt, n, r, p, dklen, maxmem): - # some random derived key - return b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb" + _fake_secret = b"this is fine" + _some_random_key = b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb" - return patch( + with patch( + "addon_service.credentials.encryption.settings.GRAVYVALET_ENCRYPT_SECRET", + _fake_secret, + ), patch( "addon_service.credentials.encryption.hashlib.scrypt", - side_effect=_mock_scrypt, - ) + return_value=_some_random_key, + ): + yield From 158d83a303aa7adceec5b4940bcedf2a40234079 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:09:36 -0400 Subject: [PATCH 04/20] fixups for deployment - allow running management commands with neither DEBUG nor GRAVYVALET_ENCRYPT_SECRET set - stop supplying a fake GRAVYVALET_ENCRYPT_SECRET in app.settings -- instead, set a fake GRAVYVALET_ENCRYPT_SECRET in docker-compose.yml --- addon_service/credentials/encryption.py | 6 ++++++ app/env.py | 2 +- app/settings.py | 10 ++-------- docker-compose.yml | 14 +++----------- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/addon_service/credentials/encryption.py b/addon_service/credentials/encryption.py index 45090944..e597fd15 100644 --- a/addon_service/credentials/encryption.py +++ b/addon_service/credentials/encryption.py @@ -132,6 +132,12 @@ def _derive_multifernet_key( key_params: KeyParameters, /, # positional-only params for cache-friendliness ) -> fernet.MultiFernet: + if not settings.GRAVYVALET_ENCRYPT_SECRET: + raise RuntimeError( + "gravyvalet can not keep your secrets without a GRAVYVALET_ENCRYPT_SECRET" + " -- ideally chosen by strong randomness, with around 256 bits of entropy" + " (e.g. 64 hex digits; 60 d20 rolls; 20 words of a 10000-word vocabulary)" + ) # https://cryptography.io/en/latest/fernet/#cryptography.fernet.MultiFernet return fernet.MultiFernet( [ diff --git a/app/env.py b/app/env.py index 8d6edda3..98a8c9bc 100644 --- a/app/env.py +++ b/app/env.py @@ -56,7 +56,7 @@ # 2. call `.rotate_encryption()` on every `ExternalCredentials` (perhaps via # celery tasks in `addon_service.tasks.key_rotation`) # 3. remove the old secret from GRAVYVALET_ENCRYPT_SECRET_PRIORS -GRAVYVALET_ENCRYPT_SECRET = os.environ.get("GRAVYVALET_ENCRYPT_SECRET") +GRAVYVALET_ENCRYPT_SECRET: str | None = os.environ.get("GRAVYVALET_ENCRYPT_SECRET") GRAVYVALET_ENCRYPT_SECRET_PRIORS = tuple( filter(bool, os.environ.get("GRAVYVALET_ENCRYPT_SECRET_PRIORS", "").split(",")) ) diff --git a/app/settings.py b/app/settings.py index 6bc9c8d1..30443488 100644 --- a/app/settings.py +++ b/app/settings.py @@ -7,14 +7,8 @@ OSF_HMAC_KEY = env.OSF_HMAC_KEY or "lmaoooooo" OSF_HMAC_EXPIRATION_SECONDS = env.OSF_HMAC_EXPIRATION_SECONDS -if not env.DEBUG and not env.GRAVYVALET_ENCRYPT_SECRET: - raise RuntimeError( - "pls set `GRAVYVALET_ENCRYPT_SECRET` environment variable to something safely random" - ) -GRAVYVALET_ENCRYPT_SECRET: bytes = ( - env.GRAVYVALET_ENCRYPT_SECRET.encode() - if env.GRAVYVALET_ENCRYPT_SECRET - else b"this is fine" +GRAVYVALET_ENCRYPT_SECRET: bytes | None = ( + env.GRAVYVALET_ENCRYPT_SECRET.encode() if env.GRAVYVALET_ENCRYPT_SECRET else None ) GRAVYVALET_ENCRYPT_SECRET_PRIORS: tuple[bytes, ...] = tuple( _prior.encode() for _prior in env.GRAVYVALET_ENCRYPT_SECRET_PRIORS diff --git a/docker-compose.yml b/docker-compose.yml index 63719bc5..3df5ea04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: build: . restart: unless-stopped command: python manage.py runserver 0.0.0.0:8004 - environment: + environment: &gv_environment DEBUG: 1 DJANGO_SETTINGS_MODULE: app.settings PYTHONUNBUFFERED: 1 @@ -20,6 +20,7 @@ services: OSF_BASE_URL: "http://192.168.168.167:5000" OSF_API_BASE_URL: "http://192.168.168.167:8000" AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + GRAVYVALET_ENCRYPT_SECRET: "this is fine" ports: - 8004:8004 stdin_open: true @@ -38,16 +39,7 @@ services: - postgres volumes: - ./:/code:cached - environment: - DEBUG: 1 - DJANGO_SETTINGS_MODULE: app.settings - POSTGRES_HOST: postgres - POSTGRES_DB: gravyvalet - POSTGRES_USER: postgres - SECRET_KEY: so-secret - OSF_BASE_URL: "http://192.168.168.167:5000" - OSF_API_BASE_URL: "http://192.168.168.167:8000" - AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + environment: *gv_environment stdin_open: true postgres: From 21b4d638c483c2330661c6b095685cbab9045a78 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:25:50 -0400 Subject: [PATCH 05/20] add STATIC_ROOT --- app/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/settings.py b/app/settings.py index 30443488..ec83c8ea 100644 --- a/app/settings.py +++ b/app/settings.py @@ -186,6 +186,7 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ +STATIC_ROOT = BASE_DIR / "static" STATIC_URL = "/static/" OSF_SENSITIVE_DATA_SECRET = env.OSF_SENSITIVE_DATA_SECRET From 03dd1322c9d4b41eb81706b6882aed57a409759c Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:37:50 -0400 Subject: [PATCH 06/20] allow testing without secrets --- addon_service/tests/_helpers.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/addon_service/tests/_helpers.py b/addon_service/tests/_helpers.py index 920f9830..53ba2299 100644 --- a/addon_service/tests/_helpers.py +++ b/addon_service/tests/_helpers.py @@ -187,13 +187,16 @@ def get_test_request(user=None, method="get", path="", cookies=None): return _request +@contextlib.contextmanager def patch_encryption_key_derivation(): - # expect to call scrypt with all the following params: - def _mock_scrypt(secret, salt, n, r, p, dklen, maxmem): - # some random derived key - return b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb" + _fake_secret = b"this is fine" + _some_random_key = b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb" - return patch( + with patch( + "addon_service.credentials.encryption.settings.GRAVYVALET_ENCRYPT_SECRET", + _fake_secret, + ), patch( "addon_service.credentials.encryption.hashlib.scrypt", - side_effect=_mock_scrypt, - ) + return_value=_some_random_key, + ): + yield From 3e0c082f07d7fdff463e68efcc753d4315058143 Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Mon, 1 Jul 2024 14:07:53 -0400 Subject: [PATCH 07/20] Temporarily disable bootsteps --- app/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/celery.py b/app/celery.py index 6010e7f7..75c3d400 100644 --- a/app/celery.py +++ b/app/celery.py @@ -68,4 +68,4 @@ def _enqueue_handler_task(body, message): ] -app.steps["consumer"].add(OsfBackchannelConsumerStep) +# app.steps["consumer"].add(OsfBackchannelConsumerStep) From 9d772d7a2c6d65d4f6e4ce4d6963f31a92194464 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:09:36 -0400 Subject: [PATCH 08/20] fixups for deployment - allow running management commands with neither DEBUG nor GRAVYVALET_ENCRYPT_SECRET set - stop supplying a fake GRAVYVALET_ENCRYPT_SECRET in app.settings -- instead, set a fake GRAVYVALET_ENCRYPT_SECRET in docker-compose.yml --- addon_service/credentials/encryption.py | 6 ++++++ app/env.py | 2 +- app/settings.py | 10 ++-------- docker-compose.yml | 14 +++----------- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/addon_service/credentials/encryption.py b/addon_service/credentials/encryption.py index 45090944..e597fd15 100644 --- a/addon_service/credentials/encryption.py +++ b/addon_service/credentials/encryption.py @@ -132,6 +132,12 @@ def _derive_multifernet_key( key_params: KeyParameters, /, # positional-only params for cache-friendliness ) -> fernet.MultiFernet: + if not settings.GRAVYVALET_ENCRYPT_SECRET: + raise RuntimeError( + "gravyvalet can not keep your secrets without a GRAVYVALET_ENCRYPT_SECRET" + " -- ideally chosen by strong randomness, with around 256 bits of entropy" + " (e.g. 64 hex digits; 60 d20 rolls; 20 words of a 10000-word vocabulary)" + ) # https://cryptography.io/en/latest/fernet/#cryptography.fernet.MultiFernet return fernet.MultiFernet( [ diff --git a/app/env.py b/app/env.py index 8d6edda3..98a8c9bc 100644 --- a/app/env.py +++ b/app/env.py @@ -56,7 +56,7 @@ # 2. call `.rotate_encryption()` on every `ExternalCredentials` (perhaps via # celery tasks in `addon_service.tasks.key_rotation`) # 3. remove the old secret from GRAVYVALET_ENCRYPT_SECRET_PRIORS -GRAVYVALET_ENCRYPT_SECRET = os.environ.get("GRAVYVALET_ENCRYPT_SECRET") +GRAVYVALET_ENCRYPT_SECRET: str | None = os.environ.get("GRAVYVALET_ENCRYPT_SECRET") GRAVYVALET_ENCRYPT_SECRET_PRIORS = tuple( filter(bool, os.environ.get("GRAVYVALET_ENCRYPT_SECRET_PRIORS", "").split(",")) ) diff --git a/app/settings.py b/app/settings.py index 6bc9c8d1..30443488 100644 --- a/app/settings.py +++ b/app/settings.py @@ -7,14 +7,8 @@ OSF_HMAC_KEY = env.OSF_HMAC_KEY or "lmaoooooo" OSF_HMAC_EXPIRATION_SECONDS = env.OSF_HMAC_EXPIRATION_SECONDS -if not env.DEBUG and not env.GRAVYVALET_ENCRYPT_SECRET: - raise RuntimeError( - "pls set `GRAVYVALET_ENCRYPT_SECRET` environment variable to something safely random" - ) -GRAVYVALET_ENCRYPT_SECRET: bytes = ( - env.GRAVYVALET_ENCRYPT_SECRET.encode() - if env.GRAVYVALET_ENCRYPT_SECRET - else b"this is fine" +GRAVYVALET_ENCRYPT_SECRET: bytes | None = ( + env.GRAVYVALET_ENCRYPT_SECRET.encode() if env.GRAVYVALET_ENCRYPT_SECRET else None ) GRAVYVALET_ENCRYPT_SECRET_PRIORS: tuple[bytes, ...] = tuple( _prior.encode() for _prior in env.GRAVYVALET_ENCRYPT_SECRET_PRIORS diff --git a/docker-compose.yml b/docker-compose.yml index 63719bc5..3df5ea04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: build: . restart: unless-stopped command: python manage.py runserver 0.0.0.0:8004 - environment: + environment: &gv_environment DEBUG: 1 DJANGO_SETTINGS_MODULE: app.settings PYTHONUNBUFFERED: 1 @@ -20,6 +20,7 @@ services: OSF_BASE_URL: "http://192.168.168.167:5000" OSF_API_BASE_URL: "http://192.168.168.167:8000" AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + GRAVYVALET_ENCRYPT_SECRET: "this is fine" ports: - 8004:8004 stdin_open: true @@ -38,16 +39,7 @@ services: - postgres volumes: - ./:/code:cached - environment: - DEBUG: 1 - DJANGO_SETTINGS_MODULE: app.settings - POSTGRES_HOST: postgres - POSTGRES_DB: gravyvalet - POSTGRES_USER: postgres - SECRET_KEY: so-secret - OSF_BASE_URL: "http://192.168.168.167:5000" - OSF_API_BASE_URL: "http://192.168.168.167:8000" - AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + environment: *gv_environment stdin_open: true postgres: From d7e3d15c42b380bdef6e77bf29ae300e9a1422b2 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:25:50 -0400 Subject: [PATCH 09/20] add STATIC_ROOT --- app/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/settings.py b/app/settings.py index 30443488..ec83c8ea 100644 --- a/app/settings.py +++ b/app/settings.py @@ -186,6 +186,7 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ +STATIC_ROOT = BASE_DIR / "static" STATIC_URL = "/static/" OSF_SENSITIVE_DATA_SECRET = env.OSF_SENSITIVE_DATA_SECRET From 3224e0d52631c144d64abadf4f60491a305440de Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 27 Jun 2024 16:37:50 -0400 Subject: [PATCH 10/20] allow testing without secrets --- addon_service/tests/_helpers.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/addon_service/tests/_helpers.py b/addon_service/tests/_helpers.py index 920f9830..53ba2299 100644 --- a/addon_service/tests/_helpers.py +++ b/addon_service/tests/_helpers.py @@ -187,13 +187,16 @@ def get_test_request(user=None, method="get", path="", cookies=None): return _request +@contextlib.contextmanager def patch_encryption_key_derivation(): - # expect to call scrypt with all the following params: - def _mock_scrypt(secret, salt, n, r, p, dklen, maxmem): - # some random derived key - return b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb" + _fake_secret = b"this is fine" + _some_random_key = b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb" - return patch( + with patch( + "addon_service.credentials.encryption.settings.GRAVYVALET_ENCRYPT_SECRET", + _fake_secret, + ), patch( "addon_service.credentials.encryption.hashlib.scrypt", - side_effect=_mock_scrypt, - ) + return_value=_some_random_key, + ): + yield From ad48d87a33fd7416e808548cb7ef1d9ede731e3a Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Mon, 1 Jul 2024 14:07:53 -0400 Subject: [PATCH 11/20] Temporarily disable bootsteps --- app/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/celery.py b/app/celery.py index 6010e7f7..75c3d400 100644 --- a/app/celery.py +++ b/app/celery.py @@ -68,4 +68,4 @@ def _enqueue_handler_task(body, message): ] -app.steps["consumer"].add(OsfBackchannelConsumerStep) +# app.steps["consumer"].add(OsfBackchannelConsumerStep) From 5426b65b889f73227d8f3dc6df831e5aae71eb5f Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Mon, 1 Jul 2024 14:52:20 -0400 Subject: [PATCH 12/20] Add `credentials_available` and `initiate_oauth` fields to ASA --- addon_service/authorized_storage_account/models.py | 4 ++++ .../authorized_storage_account/serializers.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/addon_service/authorized_storage_account/models.py b/addon_service/authorized_storage_account/models.py index 8db07988..921fc114 100644 --- a/addon_service/authorized_storage_account/models.py +++ b/addon_service/authorized_storage_account/models.py @@ -189,6 +189,10 @@ def api_base_url(self, value): def imp_cls(self) -> type[AddonImp]: return self.external_service.addon_imp.imp_cls + @property + def credentials_available(self) -> bool: + return self._credentials is not None + @transaction.atomic def initiate_oauth2_flow(self, authorized_scopes=None): if self.credentials_format is not CredentialsFormats.OAUTH2: diff --git a/addon_service/authorized_storage_account/serializers.py b/addon_service/authorized_storage_account/serializers.py index 7598df7c..89b35141 100644 --- a/addon_service/authorized_storage_account/serializers.py +++ b/addon_service/authorized_storage_account/serializers.py @@ -66,6 +66,7 @@ def __init__(self, *args, **kwargs): related_link_view_name=view_names.related_view(RESOURCE_TYPE), ) credentials = CredentialsField(write_only=True, required=False) + initiate_oauth = serializers.BooleanField(write_only=True, required=False) included_serializers = { "account_owner": "addon_service.serializers.UserReferenceSerializer", @@ -91,12 +92,16 @@ def create(self, validated_data): except ModelValidationError as e: raise serializers.ValidationError(e) - if external_service.credentials_format is CredentialsFormats.OAUTH2: + if ( + validated_data.get("initiate_oauth", False) + and external_service.credentials_format is CredentialsFormats.OAUTH2 + ): authorized_account.initiate_oauth2_flow( validated_data.get("authorized_scopes") ) - else: + elif validated_data.get("credentials"): authorized_account.credentials = validated_data["credentials"] + try: authorized_account.save() except ModelValidationError as e: @@ -120,4 +125,6 @@ class Meta: "credentials", "default_root_folder", "external_storage_service", + "initiate_oauth", + "credentials_available", ] From ef4c02d4448f66ea074b8779ec1b4a965b8474d3 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Tue, 2 Jul 2024 09:06:23 -0400 Subject: [PATCH 13/20] testfixs --- addon_service/authorized_storage_account/models.py | 5 ++++- addon_service/tests/e2e_tests/test_oauth_flow.py | 1 + .../tests/test_by_type/test_authorized_storage_account.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/addon_service/authorized_storage_account/models.py b/addon_service/authorized_storage_account/models.py index 921fc114..bdced720 100644 --- a/addon_service/authorized_storage_account/models.py +++ b/addon_service/authorized_storage_account/models.py @@ -163,7 +163,10 @@ def auth_url(self) -> str | None: Returns None if the ExternalStorageService does not support OAuth2 or if the initial credentials exchange has already ocurred. """ - if self.credentials_format is not CredentialsFormats.OAUTH2: + if ( + self.credentials_format is not CredentialsFormats.OAUTH2 + or self.oauth2_token_metadata_id is None + ): return None state_token = self.oauth2_token_metadata.state_token diff --git a/addon_service/tests/e2e_tests/test_oauth_flow.py b/addon_service/tests/e2e_tests/test_oauth_flow.py index b2dd8678..356a0338 100644 --- a/addon_service/tests/e2e_tests/test_oauth_flow.py +++ b/addon_service/tests/e2e_tests/test_oauth_flow.py @@ -26,6 +26,7 @@ def _make_post_payload(*, external_service, capabilities=None, credentials=None) "type": "authorized-storage-accounts", "attributes": { "authorized_capabilities": [AddonCapabilities.ACCESS.name], + "initiate_oauth": True, }, "relationships": { "external_storage_service": { diff --git a/addon_service/tests/test_by_type/test_authorized_storage_account.py b/addon_service/tests/test_by_type/test_authorized_storage_account.py index 7076e2f6..f6392c2d 100644 --- a/addon_service/tests/test_by_type/test_authorized_storage_account.py +++ b/addon_service/tests/test_by_type/test_authorized_storage_account.py @@ -56,6 +56,7 @@ def _make_post_payload( "attributes": { "authorized_capabilities": capabilities, "api_base_url": api_root, + "initiate_oauth": True, }, "relationships": { "external_storage_service": { From 4d90e62f5dde4037fec2b162c264609c127c1235 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Tue, 2 Jul 2024 11:59:39 -0400 Subject: [PATCH 14/20] rotate_encryption command --- addon_service/management/commands/rotate_encryption.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 addon_service/management/commands/rotate_encryption.py diff --git a/addon_service/management/commands/rotate_encryption.py b/addon_service/management/commands/rotate_encryption.py new file mode 100644 index 00000000..ba330030 --- /dev/null +++ b/addon_service/management/commands/rotate_encryption.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand + +from addon_service.tasks.key_rotation import schedule_encryption_rotation__celery + + +class Command(BaseCommand): + def handle(self, *args, **kwargs): + _task = schedule_encryption_rotation__celery.apply_async() + self.stdout.write(self.style.SUCCESS(f"scheduled task {_task}")) From 1d8a0b7400db0d121424fee0a911ad37b082c11d Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Wed, 3 Jul 2024 08:40:03 -0400 Subject: [PATCH 15/20] add sentry_sdk --- app/env.py | 29 +++++++++++++++++------------ app/settings.py | 16 ++++++++++++++++ requirements/release.txt | 1 + 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/app/env.py b/app/env.py index 98a8c9bc..a656bc6c 100644 --- a/app/env.py +++ b/app/env.py @@ -4,6 +4,17 @@ import os +DEBUG = bool(os.environ.get("DEBUG")) # any non-empty value enables debug mode +SECRET_KEY = os.environ.get("SECRET_KEY") # used by django for cryptographic signing +ALLOWED_HOSTS = list(filter(bool, os.environ.get("ALLOWED_HOSTS", "").split(","))) +CORS_ALLOWED_ORIGINS = tuple( + filter(bool, os.environ.get("CORS_ALLOWED_ORIGINS", "").split(",")) +) +SENTRY_DSN = os.environ.get("SENTRY_DSN") + +### +# databases + POSTGRES_DB = os.environ.get("POSTGRES_DB") POSTGRES_USER = os.environ.get("POSTGRES_USER") POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD") @@ -17,17 +28,20 @@ OSFDB_PORT = os.environ.get("OSFDB_PORT", "5432") OSFDB_CONN_MAX_AGE = os.environ.get("OSFDB_CONN_MAX_AGE", 0) OSFDB_SSLMODE = os.environ.get("OSFDB_SSLMODE", "prefer") + +### +# for interacting with osf + OSF_SENSITIVE_DATA_SECRET = os.environ.get("OSF_SENSITIVE_DATA_SECRET", "") OSF_SENSITIVE_DATA_SALT = os.environ.get("OSF_SENSITIVE_DATA_SALT", "") - -SECRET_KEY = os.environ.get("SECRET_KEY") OSF_HMAC_KEY = os.environ.get("OSF_HMAC_KEY") OSF_HMAC_EXPIRATION_SECONDS = int(os.environ.get("OSF_HMAC_EXPIRATION_SECONDS", 110)) - OSF_BASE_URL = os.environ.get("OSF_BASE_URL", "https://osf.example") OSF_API_BASE_URL = os.environ.get("OSF_API_BASE_URL", "https://api.osf.example") +### # amqp/celery + AMQP_BROKER_URL = os.environ.get( "AMQP_BROKER_URL", "amqp://guest:guest@192.168.168.167:5672" ) @@ -36,15 +50,6 @@ "OSF_BACKCHANNEL_QUEUE_NAME", "account_status_changes" ) -# any non-empty value enables debug mode: -DEBUG = bool(os.environ.get("DEBUG")) - -# comma-separated list: -ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(",") -CORS_ALLOWED_ORIGINS = tuple( - filter(bool, os.environ.get("CORS_ALLOWED_ORIGINS", "").split(",")) -) - ### # credentials encryption secrets and parameters # diff --git a/app/settings.py b/app/settings.py index ec83c8ea..c402baf2 100644 --- a/app/settings.py +++ b/app/settings.py @@ -1,8 +1,24 @@ +import logging from pathlib import Path from app import env +_logger = logging.getLogger(__name__) + + +if env.SENTRY_DSN: + try: + import sentry_sdk + except ImportError: + _logger.warning("SENTRY_DSN defined but sentry_sdk not installed!") + else: + sentry_sdk.init( + dsn=env.SENTRY_DSN, + environment=env.OSF_BASE_URL, + ) + + SECRET_KEY = env.SECRET_KEY OSF_HMAC_KEY = env.OSF_HMAC_KEY or "lmaoooooo" OSF_HMAC_EXPIRATION_SECONDS = env.OSF_HMAC_EXPIRATION_SECONDS diff --git a/requirements/release.txt b/requirements/release.txt index 0dbc1238..b150635c 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -1,3 +1,4 @@ -r ./requirements.txt # Requirements to be installed on server deployments +sentry-sdk==2.7.1 From d184847893433873a05f0ea3843c3b6c83e4029c Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Wed, 3 Jul 2024 11:11:53 -0400 Subject: [PATCH 16/20] reorder Dockerfile stages -- default gv-deploy --- Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index c03be65f..736f8f3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,14 @@ COPY . /code/ WORKDIR /code # END gv-base +# BEGIN gv-local +FROM gv-base as gv-local +# install dev and non-dev dependencies: +RUN pip3 install --no-cache-dir -r requirements/dev-requirements.txt +# Start the Django development server +CMD ["python", "manage.py", "runserver", "0.0.0.0:8004"] +# END gv-local + # BEGIN gv-deploy FROM gv-base as gv-deploy # install non-dev and release-only dependencies: @@ -16,11 +24,3 @@ RUN pip3 install --no-cache-dir -r requirements/release.txt RUN python manage.py collectstatic --noinput # note: no CMD in gv-deploy -- depends on deployment # END gv-deploy - -# BEGIN gv-local -FROM gv-base as gv-local -# install dev and non-dev dependencies: -RUN pip3 install --no-cache-dir -r requirements/dev-requirements.txt -# Start the Django development server -CMD ["python", "manage.py", "runserver", "0.0.0.0:8004"] -# END gv-local From 9b169a21ede8faa57a3090a0b5834afe45945213 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Wed, 3 Jul 2024 11:58:26 -0400 Subject: [PATCH 17/20] always install daphne --- requirements/dev-requirements.txt | 4 ---- requirements/requirements.txt | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 51f3d502..dc48fcbb 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -10,7 +10,3 @@ flake8 black==24.* isort pre-commit - -# ASGI server for local use -# https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/daphne/#integration-with-runserver -daphne diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 2dc1f2b2..f42b7c0e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,5 @@ Django==4.2.7 +daphne==4.1.2 psycopg>=3.1.8 djangorestframework==3.14.0 djangorestframework-jsonapi==6.1.0 From 5bf96aece5e632a26750ffcc7e47a0424b32f4ee Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Wed, 3 Jul 2024 12:20:21 -0400 Subject: [PATCH 18/20] use gv-local build target in docker-compose.yml --- docker-compose.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3df5ea04..b16c170d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,9 @@ version: "3.8" services: gravyvalet: - build: . + build: + context: . + target: gv-local restart: unless-stopped command: python manage.py runserver 0.0.0.0:8004 environment: &gv_environment @@ -31,7 +33,9 @@ services: - postgres celeryworker: - build: . + build: + context: . + target: gv-local restart: unless-stopped command: /usr/local/bin/celery --app app worker --uid daemon -l INFO depends_on: From 52a4488248eee1fc2b8675035645158945872d8f Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Wed, 3 Jul 2024 12:58:30 -0400 Subject: [PATCH 19/20] Store client secrets exclusively in envvars --- addon_service/oauth/models.py | 6 +++++- app/env.py | 8 ++++++++ app/settings.py | 6 ++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/addon_service/oauth/models.py b/addon_service/oauth/models.py index e581a15f..9f802df5 100644 --- a/addon_service/oauth/models.py +++ b/addon_service/oauth/models.py @@ -1,6 +1,7 @@ from datetime import timedelta from asgiref.sync import sync_to_async +from django.conf import settings from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ValidationError from django.db import ( @@ -28,7 +29,6 @@ class OAuth2ClientConfig(AddonsServiceBaseModel): token_endpoint_url = models.URLField(null=False) # The registered ID of the OAuth client client_id = models.CharField(null=True) - client_secret = models.CharField(null=True) class Meta: verbose_name = "OAuth2 Client Config" @@ -38,6 +38,10 @@ class Meta: def __repr__(self): return f'<{self.__class__.__qualname__}(pk="{self.pk}", auth_uri="{self.auth_uri}", client_id="{self.client_id}")>' + @property + def client_secret(self): + return settings.OAUTH_SECRETS[self.client_id] + __str__ = __repr__ diff --git a/app/env.py b/app/env.py index a656bc6c..6297cabc 100644 --- a/app/env.py +++ b/app/env.py @@ -79,3 +79,11 @@ ) # END credentials encryption secrets and parameters ### + + +### +# OAuth Client Settings +BOX_OAUTH2_CLIENT_ID: str | None = os.environ.get("BOX_CLIENT_ID", "box") +BOX_OAUTH2_CLIENT_SECRET: str | None = os.environ.get("BOX_SECRET", "box_secret") +# END OAuth Client Settings +### diff --git a/app/settings.py b/app/settings.py index c402baf2..20b08a78 100644 --- a/app/settings.py +++ b/app/settings.py @@ -215,3 +215,9 @@ AMQP_BROKER_URL = env.AMQP_BROKER_URL OSF_BACKCHANNEL_QUEUE_NAME = env.OSF_BACKCHANNEL_QUEUE_NAME GV_QUEUE_NAME_PREFIX = env.GV_QUEUE_NAME_PREFIX + + +### +# Mapping from OAuth Client IDs to Secrets +# DB should store the Client IDs to serve as a shared lookup, but secrets should live in ENVVARS +OAUTH_SECRETS = {env.BOX_OAUTH2_CLIENT_ID: env.BOX_OAUTH2_CLIENT_SECRET} From d8a4aa801f91dfd7f17cdb26298a1896ec843103 Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Wed, 3 Jul 2024 13:05:07 -0400 Subject: [PATCH 20/20] Don't set `client_secret` in factories; access via `dict.get` --- addon_service/oauth/models.py | 2 +- addon_service/tests/_factories.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/addon_service/oauth/models.py b/addon_service/oauth/models.py index 9f802df5..946cb2ce 100644 --- a/addon_service/oauth/models.py +++ b/addon_service/oauth/models.py @@ -40,7 +40,7 @@ def __repr__(self): @property def client_secret(self): - return settings.OAUTH_SECRETS[self.client_id] + return settings.OAUTH_SECRETS.get(self.client_id) __str__ = __repr__ diff --git a/addon_service/tests/_factories.py b/addon_service/tests/_factories.py index a0e7ee6f..ab149251 100644 --- a/addon_service/tests/_factories.py +++ b/addon_service/tests/_factories.py @@ -33,7 +33,6 @@ class Meta: auth_callback_url = "https://osf.example/auth/callback" token_endpoint_url = "https://api.example.com/oauth/token" client_id = factory.Faker("word") - client_secret = factory.Faker("word") class AddonOperationInvocationFactory(DjangoModelFactory):