Merge pull request #23 from GSA-TTS/dev-deployment

Setup Dev Deployment Infrastructure

.flake8

@@ -1,3 +1,4 @@
 [flake8]
 max-line-length = 88
 exclude = venv,.git,__pycache__,docs/source/conf.py,old,build,dist
+ignore = F403, F401

.github/workflows/deploy-dev.yml

@@ -0,0 +1,33 @@
+name: Deploy Dev
+
+on:
+  workflow_dispatch:
+
+jobs:
+  deploy-dev:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11.7
+
+      - name: Install poetry
+        shell: bash
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "/root/.local/bin" >> $GITHUB_PATH
+
+      - name: Deploy to dev environment
+        uses: 18f/cg-deploy-action@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: ${{ secrets.CF_ORG }}
+          cf_space: ${{ secrets.CF_SPACE }}
+          app_directory: ./app
+          push_arguments: "push -f manifest-dev.yml"

.gitignore

@@ -163,4 +163,7 @@ cython_debug/
 *.sqlite3
 
 # Development storage
-storage/
+storage/
+
+# Development task queue
+celery_broker/

.python-version

@@ -0,0 +1 @@
+3.11.7

documentation/adr/0004-hande-gis-data-processing-via-task-queue.md

@@ -0,0 +1,28 @@
+# 4. Handle GIS Data Processing via Task Queue
+
+Date: 2024-01-23
+
+## Status
+
+Accepted
+
+## Context
+
+In order to complete relatively compute-heavy and time-consuming GIS data analysis
+tasks, without affecting the user's expereince, we will need to use a task queue
+to manage these sorts of tasks.
+
+## Decision
+
+We will use Celery as the task queue for this project. Celery is open source,
+relatively straightforward to configure, and actively maintained. It also
+allows tasks to be executed concurrently across multiple workers, which is
+good for scaling.
+
+## Consequences
+
+- **Architecture**: Use of Celery will be limited to the `infrastructure`
+  directory.
+- **Choice of Broker**: In order to leverage our remote development environment's
+  services, we will use Redis as a broker, keeping in mind the [caveats](https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/redis.html#caveats)
+  highlighted in Celery's documentation.

manifest-dev.yml

@@ -0,0 +1,15 @@
+---
+applications:
+  - name: nad-ch-dev
+    buildpacks:
+      - https://github.com/cloudfoundry/python-buildpack
+    services:
+      - nad-ch-dev-postgres
+      - nad-ch-dev-redis
+      - nad-ch-dev-s3
+    random-route: true
+    memory: 256M
+    stack: cflinuxfs4
+    command: poetry run start web
+    env:
+      APP_ENV: dev_remote

nad_ch/application_context.py

@@ -1,13 +1,13 @@
-import os
 import logging
-from nad_ch.config import STORAGE_PATH
+import nad_ch.config as config
 from nad_ch.infrastructure.database import (
-    session_scope,
+    create_session_factory,
     SqlAlchemyDataProviderRepository,
     SqlAlchemyDataSubmissionRepository,
 )
 from nad_ch.infrastructure.logger import Logger
-from nad_ch.infrastructure.storage import LocalStorage
+from nad_ch.infrastructure.storage import LocalStorage, S3Storage
+from nad_ch.infrastructure.task_queue import LocalTaskQueue, RedisTaskQueue
 from tests.fakes import (
     FakeDataProviderRepository,
     FakeDataSubmissionRepository,
@@ -17,10 +17,30 @@
 
 class ApplicationContext:
     def __init__(self):
-        self._providers = SqlAlchemyDataProviderRepository(session_scope)
-        self._submissions = SqlAlchemyDataSubmissionRepository(session_scope)
-        self._logger = Logger(__name__)
-        self._storage = LocalStorage(STORAGE_PATH)
+        self._providers = self.create_provider_repository()
+        self._submissions = self.create_submission_repository()
+        self._logger = self.create_logger()
+        self._storage = self.create_storage()
+        self._task_queue = self.create_task_queue()
+
+    def create_provider_repository(self):
+        return SqlAlchemyDataProviderRepository(
+            create_session_factory(config.DATABASE_URL))
+
+    def create_submission_repository(self):
+        return SqlAlchemyDataSubmissionRepository(
+            create_session_factory(config.DATABASE_URL))
+
+    def create_logger(self):
+        return Logger(__name__)
+
+    def create_storage(self):
+        return S3Storage(config.S3_ACCESS_KEY, config.S3_SECRET_ACCESS_KEY,
+                         config.S3_REGION, config.S3_BUCKET_NAME)
+
+    def create_task_queue(self):
+        return RedisTaskQueue("task-queue", config.QUEUE_PASSWORD,
+                              config.QUEUE_HOST, config.QUEUE_PORT)
 
     @property
     def providers(self):
@@ -38,32 +58,57 @@ def logger(self):
     def storage(self):
         return self._storage
 
+    @property
+    def task_queue(self):
+        return self._task_queue
+
+
+class DevLocalApplicationContext(ApplicationContext):
+    def create_provider_repository(self):
+        return SqlAlchemyDataProviderRepository(
+            create_session_factory(config.DATABASE_URL)
+        )
+
+    def create_submission_repository(self):
+        return SqlAlchemyDataSubmissionRepository(
+            create_session_factory(config.DATABASE_URL)
+        )
+
+    def create_logger(self):
+        return Logger(__name__, logging.DEBUG)
+
+    def create_storage(self):
+        return LocalStorage(config.STORAGE_PATH)
+
+    def create_task_queue(self):
+        return LocalTaskQueue(
+            "local-task-queue", config.QUEUE_BROKER_URL, config.QUEUE_BACKEND_URL
+        )
+
 
 class TestApplicationContext(ApplicationContext):
-    def __init__(self):
-        self._providers = FakeDataProviderRepository()
-        self._submissions = FakeDataSubmissionRepository()
-        self._logger = Logger(__name__, logging.DEBUG)
-        self._storage = FakeStorage()
+    def create_provider_repository(self):
+        return FakeDataProviderRepository()
 
-    @property
-    def providers(self):
-        return self._providers
+    def create_submission_repository(self):
+        return FakeDataSubmissionRepository()
 
-    @property
-    def submissions(self):
-        return self._submissions
+    def create_logger(self):
+        return Logger(__name__, logging.DEBUG)
 
-    @property
-    def logger(self):
-        return self._logger
+    def create_storage(self):
+        return FakeStorage()
 
-    @property
-    def storage(self):
-        return self._storage
+    def create_task_queue(self):
+        return LocalTaskQueue(
+            "test-task-queue", config.QUEUE_BROKER_URL, config.QUEUE_BACKEND_URL
+        )
 
 
 def create_app_context():
-    if os.environ.get("APP_ENV") == "test":
+    if config.APP_ENV == "test":
         return TestApplicationContext()
-    return ApplicationContext()
+    elif config.APP_ENV == "dev_local":
+        return DevLocalApplicationContext()
+    else:
+        return ApplicationContext()

nad_ch/config/__init__.py

@@ -0,0 +1,7 @@
+from .base import APP_ENV
+
+
+if APP_ENV == 'dev_local' or APP_ENV == 'test':
+    from .development_local import *
+elif APP_ENV == 'dev_remote':
+    from .development_remote import *

nad_ch/config.py → nad_ch/config/base.py

@@ -1,10 +1,9 @@
-from dotenv import load_dotenv
 import os
+from dotenv import load_dotenv
 
 
 load_dotenv()
 
 
 APP_ENV = os.getenv("APP_ENV")
-DATABASE_URL = os.getenv("DATABASE_URL")
-STORAGE_PATH = os.getenv("STORAGE_PATH")
+WEB_PORT = os.getenv("WEB_PORT", 3000)

nad_ch/config/development_local.py

@@ -0,0 +1,9 @@
+import os
+from .base import *
+
+
+# Local development config
+STORAGE_PATH = os.getenv("STORAGE_PATH")
+DATABASE_URL = os.getenv("DATABASE_URL")
+QUEUE_BROKER_URL = os.getenv("QUEUE_BROKER_URL")
+QUEUE_BACKEND_URL = os.getenv("QUEUE_BACKEND_URL")

nad_ch/config/development_remote.py

@@ -0,0 +1,30 @@
+import json
+import os
+from .base import *
+
+
+def get_credentials(service_name, default={}):
+    service = vcap_services.get(service_name, [default])
+    return service[0].get("credentials", default) if service else default
+
+
+# Remote development config
+vcap_services = json.loads(os.getenv("VCAP_SERVICES", "{}"))
+
+
+postgres_credentials = get_credentials("aws-rds")
+redis_credentials = get_credentials("aws-elasticache-redis")
+s3_credentials = get_credentials("s3")
+
+
+DATABASE_URL = postgres_credentials.get("uri", os.getenv("DATABASE_URL"))
+QUEUE_HOST = redis_credentials.get("hostname", os.getenv("QUEUE_HOST"))
+QUEUE_PORT = redis_credentials.get("port", os.getenv("QUEUE_PORT"))
+QUEUE_PASSWORD = redis_credentials.get("password", os.getenv("QUEUE_PASSWORD"))
+S3_BUCKET_NAME = s3_credentials.get("bucket", os.getenv("S3_BUCKET_NAME"))
+S3_ENDPOINT = s3_credentials.get("endpoint", os.getenv("S3_ENDPOINT"))
+S3_ACCESS_KEY = s3_credentials.get("access_key_id", os.getenv("S3_ACCESS_KEY"))
+S3_SECRET_ACCESS_KEY = s3_credentials.get(
+    "secret_access_key", os.getenv("S3_SECRET_ACCESS_KEY")
+)
+S3_REGION = s3_credentials.get("region", os.getenv("S3_REGION"))

nad_ch/infrastructure/database.py

@@ -1,20 +1,20 @@
 from typing import List, Optional
 from sqlalchemy import Column, Integer, String, create_engine, ForeignKey, DateTime
-from sqlalchemy.orm import sessionmaker, declarative_base, relationship
+from sqlalchemy.orm import sessionmaker, declarative_base, relationship, Session
 from sqlalchemy.sql import func
 import contextlib
-from nad_ch.config import DATABASE_URL
 from nad_ch.domain.entities import DataProvider, DataSubmission
 from nad_ch.domain.repositories import DataProviderRepository, DataSubmissionRepository
 
 
-engine = create_engine(DATABASE_URL)
-Session = sessionmaker(bind=engine)
+def create_session_factory(connection_string: str):
+    engine = create_engine(connection_string)
+    return sessionmaker(bind=engine)
 
 
 @contextlib.contextmanager
-def session_scope():
-    session = Session()
+def session_scope(session_factory):
+    session = session_factory
     try:
         yield session
         session.commit()
@@ -82,9 +82,7 @@ def from_entity(submission):
         return model
 
     def to_entity(self, provider: DataProvider):
-        entity = DataSubmission(
-            id=self.id, filename=self.filename, provider=provider
-        )
+        entity = DataSubmission(id=self.id, filename=self.filename, provider=provider)
 
         if self.created_at is not None:
             entity.set_created_at(self.created_at)
@@ -96,8 +94,8 @@ def to_entity(self, provider: DataProvider):
 
 
 class SqlAlchemyDataProviderRepository(DataProviderRepository):
-    def __init__(self, session_factory):
-        self.session_factory = session_factory
+    def __init__(self, session: Session):
+        self.session_factory = session
 
     def add(self, provider: DataProvider) -> DataProvider:
         with self.session_factory() as session:
@@ -124,8 +122,8 @@ def get_all(self) -> List[DataProvider]:
 
 
 class SqlAlchemyDataSubmissionRepository(DataSubmissionRepository):
-    def __init__(self, session_factory):
-        self.session_factory = session_factory
+    def __init__(self, session: Session):
+        self.session_factory = session
 
     def add(self, submission: DataSubmission) -> DataSubmission:
         with self.session_factory() as session: