Merge pull request #101 from NASA-IMPACT/feature/circle-ci-lambda-dep…

…loyment

Implement CircleCI deployment when pushing to develop

.circleci/config.yml

@@ -18,40 +18,49 @@ jobs:
           name: run pre-commit
           command: ~/.local/bin/pre-commit run --all-files
 
-  deploy:
-    docker:
-      - image: nikolaik/python-nodejs:python3.7-nodejs12
+  deploy-staging:
+    machine: 
+      image: ubuntu-2004:202010-01
     working_directory: ~/covid-api
     steps:
       - checkout
-      - setup_remote_docker
+      - run: 
+          name: use python 3
+          command: |
+            pyenv global 3.8.5
+
       - run:
           name: install dependencies
           command: |
             pip install -e .["deploy"] --user
             npm install -g cdk
 
-      - run:
-          name: install docker-ce-cli
+      - deploy:
+          name: develop branch deployed to staging cdk stack            
           command: |
-            apt-get update
-            apt-get install apt-transport-https \
-              ca-certificates curl gnupg2 software-properties-common -y
-            curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
-            add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian buster stable"
-            apt-get update
-            apt-get install docker-ce-cli -y
+            if [ "${CIRCLE_BRANCH}" == "develop" ]; then
+              STAGE='staging' VPC_ID='vpc-0fa3007e738c7bbdf' cdk deploy covid-api-lambda-staging --region us-east-1 --require-approval never
+            fi
+
+  deploy-production:
+    machine: 
+      image: ubuntu-2004:202010-01
+    working_directory: ~/covid-api
+    steps:
+      - checkout
+      - run: 
+          name: use python 3
+          command: |
+            pyenv global 3.8.5
 
       - run:
-          name: create lambda package
+          name: install dependencies
           command: |
-            docker build . -t lambda:latest -f Dockerfiles/lambda/Dockerfile
-            docker run --name lambda lambda:latest echo "container up"
-            docker cp lambda:/tmp/package.zip package.zip
-            docker stop lambda
+            pip install -e .["deploy"] --user
+            npm install -g cdk
 
       - deploy:
-          name: master branch deployed to cdk stack
+          name: master branch deployed to production cdk stack
           command: |
             if [ "${CIRCLE_BRANCH}" == "master" ]; then
               STAGE='prod' cdk deploy covid-api-lambda-prod --region us-east-1 --require-approval never
@@ -62,7 +71,14 @@ workflows:
   test_and_deploy:
     jobs:
       - test
-      - deploy:
+      - deploy-staging: 
+          requires:
+            - test
+          filters:
+            branches:
+              # only: /^feature\/.*/
+              only: develop
+      - deploy-production:
           requires:
             - test
           filters:

Dockerfiles/lambda/Dockerfile

@@ -21,4 +21,3 @@ RUN rm -rdf /var/task/stack
 
 
 COPY lambda/handler.py /var/task/handler.py
-

cdk.context.json

@@ -0,0 +1,45 @@
+{
+  "vpc-provider:account=853558080719:filter.vpc-id=vpc-0fa3007e738c7bbdf:region=us-east-1:returnAsymmetricSubnets=true": {
+    "vpcId": "vpc-0fa3007e738c7bbdf",
+    "vpcCidrBlock": "10.0.0.0/16",
+    "availabilityZones": [],
+    "subnetGroups": [
+      {
+        "name": "Private",
+        "type": "Private",
+        "subnets": [
+          {
+            "subnetId": "subnet-04bc4ca3d119f6f6b",
+            "cidr": "10.0.128.0/18",
+            "availabilityZone": "us-east-1a",
+            "routeTableId": "rtb-0a01309e2f528c2bd"
+          },
+          {
+            "subnetId": "subnet-0bcd0f2d9b9ac1c56",
+            "cidr": "10.0.192.0/18",
+            "availabilityZone": "us-east-1b",
+            "routeTableId": "rtb-05251cbc837438e6c"
+          }
+        ]
+      },
+      {
+        "name": "Public",
+        "type": "Public",
+        "subnets": [
+          {
+            "subnetId": "subnet-009875640f64d198a",
+            "cidr": "10.0.0.0/18",
+            "availabilityZone": "us-east-1a",
+            "routeTableId": "rtb-0b1d4d54a9d962398"
+          },
+          {
+            "subnetId": "subnet-0e033da6876bf7014",
+            "cidr": "10.0.64.0/18",
+            "availabilityZone": "us-east-1b",
+            "routeTableId": "rtb-074500f27775c6bda"
+          }
+        ]
+      }
+    ]
+  }
+}

stack/app.py

@@ -48,15 +48,18 @@ def __init__(
         memory: int = 1024,
         timeout: int = 30,
         concurrent: int = 100,
-        env: dict = {},
         code_dir: str = "./",
         **kwargs: Any,
     ) -> None:
         """Define stack."""
-        super().__init__(scope, id, *kwargs)
+        super().__init__(scope, id, **kwargs)
 
         # add cache
-        vpc = ec2.Vpc(self, f"{id}-vpc")
+        if config.VPC_ID:
+            vpc = ec2.Vpc.from_lookup(self, f"{id}-vpc", vpc_id=config.VPC_ID,)
+        else:
+            vpc = ec2.Vpc(self, f"{id}-vpc")
+
         sb_group = escache.CfnSubnetGroup(
             self,
             f"{id}-subnet-group",
@@ -149,14 +152,18 @@ def __init__(
         memory: Union[int, float] = 512,
         mincount: int = 1,
         maxcount: int = 50,
-        env: dict = {},
+        task_env: dict = {},
         code_dir: str = "./",
         **kwargs: Any,
     ) -> None:
         """Define stack."""
-        super().__init__(scope, id, *kwargs)
+        super().__init__(scope, id, **kwargs)
 
-        vpc = ec2.Vpc(self, f"{id}-vpc", max_azs=2)
+        # add cache
+        if config.VPC_ID:
+            vpc = ec2.Vpc.from_lookup(self, f"{id}-vpc", vpc_id=config.VPC_ID,)
+        else:
+            vpc = ec2.Vpc(self, f"{id}-vpc")
 
         cluster = ecs.Cluster(self, f"{id}-cluster", vpc=vpc)
 
@@ -169,7 +176,7 @@ def __init__(
                 LOG_LEVEL="error",
             )
         )
-        task_env.update(env)
+        task_env.update(task_env)
 
         fargate_service = ecs_patterns.ApplicationLoadBalancedFargateService(
             self,
@@ -238,7 +245,11 @@ def __init__(
     memory=config.TASK_MEMORY,
     mincount=config.MIN_ECS_INSTANCES,
     maxcount=config.MAX_ECS_INSTANCES,
-    env=config.ENV,
+    task_env=config.TASK_ENV,
+    env=dict(
+        account=os.environ["CDK_DEFAULT_ACCOUNT"],
+        region=os.environ["CDK_DEFAULT_REGION"],
+    ),
 )
 
 lambda_stackname = f"{config.PROJECT_NAME}-lambda-{config.STAGE}"
@@ -248,7 +259,10 @@ def __init__(
     memory=config.MEMORY,
     timeout=config.TIMEOUT,
     concurrent=config.MAX_CONCURRENT,
-    env=config.ENV,
+    env=dict(
+        account=os.environ["CDK_DEFAULT_ACCOUNT"],
+        region=os.environ["CDK_DEFAULT_REGION"],
+    ),
 )
 
 app.synth()

stack/config.py

@@ -9,7 +9,12 @@
 BUCKET = "covid-eo-data"
 
 # Additional environement variable to set in the task/lambda
-ENV: dict = dict()
+TASK_ENV: dict = dict()
+
+# Existing VPC to point ECS/LAMBDA stacks towards. Defaults to creating a new
+# VPC if no ID is supplied.
+VPC_ID = os.environ.get("VPC_ID")
+
 
 ################################################################################
 #                                                                              #