Merge branch 'develop' of https://github.com/developmentseed/titiler-cmr

 into ab/initial-api-spec

.github/workflows/ci.yml

@@ -0,0 +1,112 @@
+name: Test and Deploy
+
+# Triggers on pushes to main, dev and tags.
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - main
+    - develop
+    tags:
+    - 'v*'
+    paths:
+      # Only run test and docker publish if some code have changed
+      - 'pyproject.toml'
+      - 'infrastructure/aws/**'
+      - 'titiler/**'
+      - '.pre-commit-config.yaml'
+      - '.github/workflows/ci.yml'
+
+  # Run tests on pull requests.
+  pull_request:
+env:
+  LATEST_PY_VERSION: '3.10'
+
+permissions:
+  id-token: write   # This is required for requesting the JWT
+  contents: read    # This is required for actions/checkout
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .["test"]
+
+      - name: run pre-commit
+        if: ${{ matrix.python-version == env.LATEST_PY_VERSION }}
+        run: |
+          python -m pip install pre-commit
+          pre-commit run --all-files
+
+      - name: Run tests
+        run: python -m pytest --cov titiler.cmr --cov-report term-missing -s -vv
+
+  deploy:
+    needs: [tests]
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')
+
+    defaults:
+      run:
+        working-directory: infrastructure/aws
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.deploy_role_arn }}
+          role-session-name: samplerolesession
+          aws-region: us-west-2
+
+      - name: Set up node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14.x'
+
+      - name: Install cdk
+        run: npm install -g
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements-cdk.txt
+
+      # Build and deploy to the development environment whenever there is a push to main or dev
+      - name: Build & Deploy Development
+        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop'
+        run: npm run cdk -- deploy titiler-cmr-staging --require-approval never
+        env:
+          # STACK_ALARM_EMAIL: ${{ secrets.ALARM_EMAIL }}
+          STACK_ROLE_ARN: ${{ secrets.lambda_role_arn }}
+          STACK_STAGE: staging
+          STACK_ADDITIONAL_ENV: '{"TITILER_CMR_S3_AUTH_STRATEGY":"iam", "TITILER_CMR_API_DEBUG":"TRUE"}'
+
+      # Build and deploy to production deployment whenever there a new tag is pushed
+      - name: Build & Deploy Production
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: npm run cdk -- deploy titiler-cmr-production --require-approval never
+        env:
+          # STACK_ALARM_EMAIL: ${{ secrets.ALARM_EMAIL }}
+          STACK_ROLE_ARN: ${{ secrets.lambda_role_arn }}
+          STACK_STAGE: production
+          STACK_ADDITIONAL_ENV: '{"TITILER_CMR_S3_AUTH_STRATEGY":"iam"}'

.gitignore

@@ -100,3 +100,8 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# AWS CDK
+cdk.out/
+node_modules
+cdk.context.json

README.md

@@ -1,7 +1,3 @@
 # titiler-cmr
 
 An API for creating image tiles from CMR queries.
-
-## What's here
-
-[`/stac/collections/`](`/stac/collections/`) contains STAC collections json. Requests to titiler-cmr should contain all the necessary parameters to query CMR and open a dataset. At this time, that information will be stored as STAC. These files contain examples of what those STAC entries for CMR will look like. But it will be up to clients (such as a UI) to fetch STAC entries and parse them to pass a request to titiler.

stac/collections/mursst-cmr.json → data/mursst-cmr.json

@@ -1,5 +1,5 @@
 {
-    "id": "MUR-JPL-L4-GLOB-v4.1.v4.1",
+    "id": "MUR-JPL-L4-GLOB",
     "stac_version": "1.0.0",
     "license": "not-provided",
     "title": "GHRSST Level 4 MUR Global Foundation Sea Surface Temperature Analysis (v4.1)",
@@ -23,7 +23,7 @@
             "lon"
           ],
           "type": "data"
-        }      
+        }
     },
     "renders": {
       "analysed_sst": {
@@ -34,7 +34,7 @@
         "resampling": "average",
         "colormap_name": "ocean"
       }
-    },  
+    },
     "links": [
       {
         "rel": "self",
@@ -70,4 +70,3 @@
       }
     }
   }
-

infrastructure/aws/cdk.json

@@ -0,0 +1,3 @@
+{
+    "app": "python3 cdk/app.py"
+}

infrastructure/aws/cdk/__init__.py

@@ -0,0 +1 @@
+"""AWS App."""

infrastructure/aws/cdk/app.py

@@ -0,0 +1,163 @@
+"""Construct App."""
+
+import os
+from typing import Any, Dict, List, Optional
+
+from aws_cdk import App, CfnOutput, Duration, Stack, Tags
+from aws_cdk import aws_apigatewayv2_alpha as apigw
+from aws_cdk import aws_cloudwatch as cloudwatch
+from aws_cdk import aws_cloudwatch_actions as cloudwatch_actions
+from aws_cdk import aws_iam as iam
+from aws_cdk import aws_lambda
+from aws_cdk import aws_logs as logs
+from aws_cdk import aws_sns as sns
+from aws_cdk import aws_sns_subscriptions as subscriptions
+from aws_cdk.aws_apigatewayv2_integrations_alpha import HttpLambdaIntegration
+from config import StackSettings
+from constructs import Construct
+
+settings = StackSettings()
+
+DEFAULT_ENV = {
+    "GDAL_CACHEMAX": "200",  # 200 mb
+    "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
+    "GDAL_INGESTED_BYTES_AT_OPEN": "32768",  # get more bytes when opening the files.
+    "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES",
+    "GDAL_HTTP_MULTIPLEX": "YES",
+    "GDAL_HTTP_VERSION": "2",
+    "PYTHONWARNINGS": "ignore",
+    "VSI_CACHE": "TRUE",
+    "VSI_CACHE_SIZE": "5000000",  # 5 MB (per file-handle)
+}
+
+
+class LambdaStack(Stack):
+    """Lambda Stack"""
+
+    def __init__(
+        self,
+        scope: Construct,
+        id: str,
+        memory: int = 1024,
+        timeout: int = 30,
+        runtime: aws_lambda.Runtime = aws_lambda.Runtime.PYTHON_3_10,
+        concurrent: Optional[int] = None,
+        permissions: Optional[List[iam.PolicyStatement]] = None,
+        environment: Optional[Dict] = None,
+        role_arn: Optional[str] = None,
+        context_dir: str = "../../",
+        **kwargs: Any,
+    ) -> None:
+        """Define stack."""
+        super().__init__(scope, id, *kwargs)
+
+        permissions = permissions or []
+        environment = environment or {}
+
+        iam_reader_role = None
+        if role_arn:
+            iam_reader_role = iam.Role.from_role_arn(
+                self,
+                "veda-reader-dev-role",
+                role_arn=role_arn,
+            )
+
+        lambda_function = aws_lambda.Function(
+            self,
+            f"{id}-lambda",
+            runtime=runtime,
+            code=aws_lambda.Code.from_docker_build(
+                path=os.path.abspath(context_dir),
+                file="infrastructure/aws/lambda/Dockerfile",
+                platform="linux/amd64",
+            ),
+            handler="handler.handler",
+            memory_size=memory,
+            reserved_concurrent_executions=concurrent,
+            timeout=Duration.seconds(timeout),
+            environment={
+                **DEFAULT_ENV,
+                **environment,
+            },
+            log_retention=logs.RetentionDays.ONE_WEEK,
+            role=iam_reader_role,
+        )
+
+        for perm in permissions:
+            lambda_function.add_to_role_policy(perm)
+
+        api = apigw.HttpApi(
+            self,
+            f"{id}-endpoint",
+            default_integration=HttpLambdaIntegration(
+                f"{id}-integration", lambda_function
+            ),
+        )
+
+        # Create an SNS Topic
+        if settings.alarm_email:
+            topic = sns.Topic(
+                self,
+                f"{id}-500-Errors",
+                display_name=f"{id} Gateway 500 Errors",
+                topic_name=f"{id}-Gateway-500-Errors",
+            )
+            # Subscribe email to the topic
+            topic.add_subscription(
+                subscriptions.EmailSubscription(settings.alarm_email),
+            )
+
+            # Create CloudWatch Alarm
+            alarm = cloudwatch.Alarm(
+                self,
+                "MyAlarm",
+                metric=cloudwatch.Metric(
+                    namespace="AWS/ApiGateway",
+                    metric_name="5XXError",
+                    dimensions_map={"ApiName": f"{id}-endpoint"},
+                    period=Duration.minutes(1),
+                ),
+                evaluation_periods=1,
+                threshold=1,
+                alarm_description="Alarm if 500 errors are detected",
+                alarm_name=f"{id}-ApiGateway500Alarm",
+                actions_enabled=True,
+            )
+            alarm.add_alarm_action(cloudwatch_actions.SnsAction(topic))
+
+        CfnOutput(self, "Endpoint", value=api.url)
+
+
+app = App()
+
+perms = []
+if settings.buckets:
+    perms.append(
+        iam.PolicyStatement(
+            actions=["s3:GetObject"],
+            resources=[f"arn:aws:s3:::{bucket}*" for bucket in settings.buckets],
+        )
+    )
+
+lambda_stack = LambdaStack(
+    app,
+    f"{settings.name}-{settings.stage}",
+    memory=settings.memory,
+    timeout=settings.timeout,
+    concurrent=settings.max_concurrent,
+    role_arn=settings.role_arn,
+    permissions=perms,
+    environment=settings.additional_env,
+)
+# Tag infrastructure
+for key, value in {
+    "Project": settings.name,
+    "Stack": settings.stage,
+    "Owner": settings.owner,
+    "Client": settings.client,
+}.items():
+    if value:
+        Tags.of(lambda_stack).add(key, value)
+
+
+app.synth()

infrastructure/aws/cdk/config.py

@@ -0,0 +1,41 @@
+"""STACK Configs."""
+
+from typing import Dict, List, Optional
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class StackSettings(BaseSettings):
+    """Application settings"""
+
+    name: str = "titiler-cmr"
+    stage: str = "production"
+
+    owner: Optional[str] = None
+    client: Optional[str] = None
+    project: Optional[str] = None
+
+    additional_env: Dict = {}
+
+    # S3 bucket names where TiTiler could do HEAD and GET Requests
+    # specific private and public buckets MUST be added if you want to use s3:// urls
+    # You can whitelist all bucket by setting `*`.
+    # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html
+    buckets: List = []
+
+    # S3 key pattern to limit the access to specific items (e.g: "my_data/*.tif")
+    key: str = "*"
+
+    timeout: int = 30
+    memory: int = 3009
+
+    role_arn: Optional[str] = None
+
+    # The maximum of concurrent executions you want to reserve for the function.
+    # Default: - No specific limit - account limit.
+    max_concurrent: Optional[int] = None
+    alarm_email: Optional[str] = None
+
+    model_config = SettingsConfigDict(
+        env_prefix="STACK_", env_file=".env", extra="ignore"
+    )