Merge pull request #11 from developmentseed/feature/add-infrastructur…

…e-code

add cdk code

.gitignore

@@ -100,3 +100,8 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# AWS CDK
+cdk.out/
+node_modules
+cdk.context.json

infrastructure/aws/cdk.json

@@ -0,0 +1,3 @@
+{
+    "app": "python3 cdk/app.py"
+}

infrastructure/aws/cdk/__init__.py

@@ -0,0 +1 @@
+"""AWS App."""

infrastructure/aws/cdk/app.py

@@ -0,0 +1,164 @@
+"""Construct App."""
+
+import os
+from typing import Any, Dict, List, Optional
+
+from aws_cdk import App, CfnOutput, Duration, Stack, Tags
+from aws_cdk import aws_apigatewayv2_alpha as apigw
+from aws_cdk import aws_cloudwatch as cloudwatch
+from aws_cdk import aws_cloudwatch_actions as cloudwatch_actions
+from aws_cdk import aws_iam as iam
+from aws_cdk import aws_lambda
+from aws_cdk import aws_logs as logs
+from aws_cdk import aws_sns as sns
+from aws_cdk import aws_sns_subscriptions as subscriptions
+from aws_cdk.aws_apigatewayv2_integrations_alpha import HttpLambdaIntegration
+from config import StackSettings
+from constructs import Construct
+
+settings = StackSettings()
+
+
+DEFAULT_ENV = {
+    "GDAL_CACHEMAX": "200",  # 200 mb
+    "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
+    "GDAL_INGESTED_BYTES_AT_OPEN": "32768",  # get more bytes when opening the files.
+    "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES",
+    "GDAL_HTTP_MULTIPLEX": "YES",
+    "GDAL_HTTP_VERSION": "2",
+    "PYTHONWARNINGS": "ignore",
+    "VSI_CACHE": "TRUE",
+    "VSI_CACHE_SIZE": "5000000",  # 5 MB (per file-handle)
+}
+
+
+class LambdaStack(Stack):
+    """Lambda Stack"""
+
+    def __init__(
+        self,
+        scope: Construct,
+        id: str,
+        memory: int = 1024,
+        timeout: int = 30,
+        runtime: aws_lambda.Runtime = aws_lambda.Runtime.PYTHON_3_11,
+        concurrent: Optional[int] = None,
+        permissions: Optional[List[iam.PolicyStatement]] = None,
+        environment: Optional[Dict] = None,
+        role_arn: Optional[str] = None,
+        context_dir: str = "../../",
+        **kwargs: Any,
+    ) -> None:
+        """Define stack."""
+        super().__init__(scope, id, *kwargs)
+
+        permissions = permissions or []
+        environment = environment or {}
+
+        if role_arn:
+            iam_reader_role = iam.Role.from_role_arn(
+                self,
+                "veda-reader-dev-role",
+                role_arn=role_arn,
+            )
+
+        lambda_function = aws_lambda.Function(
+            self,
+            f"{id}-lambda",
+            runtime=runtime,
+            code=aws_lambda.Code.from_docker_build(
+                path=os.path.abspath(context_dir),
+                file="infrastructure/aws/lambda/Dockerfile",
+                platform="linux/amd64",
+            ),
+            handler="handler.handler",
+            memory_size=memory,
+            reserved_concurrent_executions=concurrent,
+            timeout=Duration.seconds(timeout),
+            environment={
+                **DEFAULT_ENV,
+                **environment,
+            },
+            log_retention=logs.RetentionDays.ONE_WEEK,
+            role=iam_reader_role,
+        )
+
+        for perm in permissions:
+            lambda_function.add_to_role_policy(perm)
+
+        api = apigw.HttpApi(
+            self,
+            f"{id}-endpoint",
+            default_integration=HttpLambdaIntegration(
+                f"{id}-integration", lambda_function
+            ),
+        )
+
+        # Create an SNS Topic
+        if settings.alarm_email:
+            topic = sns.Topic(
+                self,
+                f"{id}-500-Errors",
+                display_name=f"{id} Gateway 500 Errors",
+                topic_name=f"{id}-Gateway-500-Errors",
+            )
+            # Subscribe email to the topic
+            topic.add_subscription(
+                subscriptions.EmailSubscription(settings.alarm_email),
+            )
+
+            # Create CloudWatch Alarm
+            alarm = cloudwatch.Alarm(
+                self,
+                "MyAlarm",
+                metric=cloudwatch.Metric(
+                    namespace="AWS/ApiGateway",
+                    metric_name="5XXError",
+                    dimensions_map={"ApiName": f"{id}-endpoint"},
+                    period=Duration.minutes(1),
+                ),
+                evaluation_periods=1,
+                threshold=1,
+                alarm_description="Alarm if 500 errors are detected",
+                alarm_name=f"{id}-ApiGateway500Alarm",
+                actions_enabled=True,
+            )
+            alarm.add_alarm_action(cloudwatch_actions.SnsAction(topic))
+
+        CfnOutput(self, "Endpoint", value=api.url)
+
+
+app = App()
+
+perms = []
+if settings.buckets:
+    perms.append(
+        iam.PolicyStatement(
+            actions=["s3:GetObject"],
+            resources=[f"arn:aws:s3:::{bucket}*" for bucket in settings.buckets],
+        )
+    )
+
+
+lambda_stack = LambdaStack(
+    app,
+    f"{settings.name}-{settings.stage}",
+    memory=settings.memory,
+    timeout=settings.timeout,
+    concurrent=settings.max_concurrent,
+    role_arn=settings.role_arn,
+    permissions=perms,
+    environment=settings.additional_env,
+)
+# Tag infrastructure
+for key, value in {
+    "Project": settings.name,
+    "Stack": settings.stage,
+    "Owner": settings.owner,
+    "Client": settings.client,
+}.items():
+    if value:
+        Tags.of(lambda_stack).add(key, value)
+
+
+app.synth()

infrastructure/aws/cdk/config.py

@@ -0,0 +1,43 @@
+"""STACK Configs."""
+
+from typing import Dict, List, Optional
+
+from pydantic_settings import BaseSettings
+
+
+class StackSettings(BaseSettings):
+    """Application settings"""
+
+    name: str = "titiler-cmr"
+    stage: str = "production"
+
+    owner: Optional[str] = None
+    client: Optional[str] = None
+    project: Optional[str] = None
+
+    additional_env: Dict = {}
+
+    # S3 bucket names where TiTiler could do HEAD and GET Requests
+    # specific private and public buckets MUST be added if you want to use s3:// urls
+    # You can whitelist all bucket by setting `*`.
+    # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html
+    buckets: List = []
+
+    # S3 key pattern to limit the access to specific items (e.g: "my_data/*.tif")
+    key: str = "*"
+
+    timeout: int = 30
+    memory: int = 3009
+
+    role_arn: Optional[str] = None
+
+    # The maximum of concurrent executions you want to reserve for the function.
+    # Default: - No specific limit - account limit.
+    max_concurrent: Optional[int] = None
+    alarm_email: Optional[str] = None
+
+    model_config = {
+        "env_prefix": "STACK",
+        "env_file": ".env",
+        "extra": "ignore",
+    }

infrastructure/aws/lambda/Dockerfile

@@ -0,0 +1,32 @@
+ARG PYTHON_VERSION=3.11
+
+FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
+
+WORKDIR /tmp
+
+COPY pyproject.toml pyproject.toml
+COPY LICENSE LICENSE
+COPY README.md README.md
+COPY titiler/ titiler/
+
+# Install dependencies
+# HACK: aiobotocore has a tight botocore dependency
+# https://github.com/aio-libs/aiobotocore/issues/862
+# and becaise we NEED to remove both boto3 and botocore to save space for the package
+# we have to force using old package version that seems `almost` compatible with Lambda env botocore
+# https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
+RUN python -m pip install --upgrade pip
+RUN python -m pip install . "mangum>=0.10.0" "botocore==1.29.76" "aiobotocore==2.5.0" -t /asset --no-binary pydantic
+
+# Reduce package size and remove useless files
+RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
+RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
+RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
+RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+RUN rm -rdf /asset/numpy/doc/ /asset/bin /asset/geos_license /asset/Misc
+RUN rm -rdf /asset/boto3*
+RUN rm -rdf /asset/botocore*
+
+COPY infrastructure/aws/lambda/handler.py /asset/handler.py
+
+CMD ["echo", "hello world"]

infrastructure/aws/lambda/handler.py

@@ -0,0 +1,12 @@
+"""AWS Lambda handler."""
+
+import logging
+
+from mangum import Mangum
+
+from titiler.cmr.main import app
+
+logging.getLogger("mangum.lifespan").setLevel(logging.ERROR)
+logging.getLogger("mangum.http").setLevel(logging.ERROR)
+
+handler = Mangum(app, lifespan="off")

infrastructure/aws/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "cdk-deploy",
+  "version": "0.1.0",
+  "description": "Dependencies for CDK deployment",
+  "license": "MIT",
+  "private": true,
+  "dependencies": {
+    "cdk": "2.76.0-alpha.0"
+  },
+  "scripts": {
+    "cdk": "cdk"
+  }
+}

infrastructure/aws/requirements-cdk.txt

@@ -0,0 +1,7 @@
+aws-cdk-lib==2.76.0
+aws_cdk-aws_apigatewayv2_alpha==2.76.0a0
+aws_cdk-aws_apigatewayv2_integrations_alpha==2.76.0a0
+constructs>=10.0.0
+
+pydantic>=2.4,<3.0
+pydantic-settings~=2.0