Merge pull request #107 from NASA-IMPACT/develop

Release 0.4.0

.circleci/config.yml

@@ -18,42 +18,53 @@ jobs:
           name: run pre-commit
           command: ~/.local/bin/pre-commit run --all-files
 
-  deploy:
-    docker:
-      - image: nikolaik/python-nodejs:python3.7-nodejs12
+  deploy-staging:
+    machine: 
+      image: ubuntu-2004:202010-01
     working_directory: ~/covid-api
     steps:
       - checkout
-      - setup_remote_docker
+      - run: 
+          name: use python 3
+          command: |
+            pyenv global 3.8.5
+
       - run:
           name: install dependencies
           command: |
             pip install -e .["deploy"] --user
             npm install -g cdk
 
-      - run:
-          name: install docker-ce-cli
+      - deploy:
+          name: develop branch deployed to staging cdk stack            
           command: |
-            apt-get update
-            apt-get install apt-transport-https \
-              ca-certificates curl gnupg2 software-properties-common -y
-            curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
-            add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian buster stable"
-            apt-get update
-            apt-get install docker-ce-cli -y
+            if [ "${CIRCLE_BRANCH}" == "develop" ]; then
+              STAGE='staging' cdk deploy covid-api-dataset-metadata-generator-staging --region us-east-1 --require-approval never  
+              STAGE='staging' VPC_ID='vpc-0fa3007e738c7bbdf' cdk deploy covid-api-lambda-staging --region us-east-1 --require-approval never
+            fi
+
+  deploy-production:
+    machine: 
+      image: ubuntu-2004:202010-01
+    working_directory: ~/covid-api
+    steps:
+      - checkout
+      - run: 
+          name: use python 3
+          command: |
+            pyenv global 3.8.5
 
       - run:
-          name: create lambda package
+          name: install dependencies
           command: |
-            docker build . -t lambda:latest -f Dockerfiles/lambda/Dockerfile
-            docker run --name lambda lambda:latest echo "container up"
-            docker cp lambda:/tmp/package.zip package.zip
-            docker stop lambda
+            pip install -e .["deploy"] --user
+            npm install -g cdk
 
       - deploy:
-          name: master branch deployed to cdk stack
+          name: master branch deployed to production cdk stack
           command: |
             if [ "${CIRCLE_BRANCH}" == "master" ]; then
+              STAGE='prod' cdk deploy covid-api-dataset-metadata-generator-prod --region us-east-1 --require-approval never  
               STAGE='prod' cdk deploy covid-api-lambda-prod --region us-east-1 --require-approval never
             fi
 
@@ -62,7 +73,14 @@ workflows:
   test_and_deploy:
     jobs:
       - test
-      - deploy:
+      - deploy-staging: 
+          requires:
+            - test
+          filters:
+            branches:
+              # only: /^feature\/.*/
+              only: develop
+      - deploy-production:
           requires:
             - test
           filters:

Dockerfiles/ecs/Dockerfile

@@ -1,4 +1,4 @@
-FROM laurents/uvicorn-gunicorn-fastapi:python3.7-slim
+FROM tiangolo/uvicorn-gunicorn:python3.8
 # Ref https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker/issues/15
 # Cuts image size by 50%
 # FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7
@@ -10,4 +10,5 @@ COPY covid_api/ /app/covid_api/
 COPY setup.py /app/setup.py
 
 RUN pip install -e /app/. --no-cache-dir
+
 CMD ["/start-reload.sh"]

Dockerfiles/lambda/Dockerfile

@@ -1,14 +1,15 @@
 FROM lambci/lambda:build-python3.7
 
-WORKDIR /tmp
 
 # We install covid_api and mangum
+WORKDIR /app
 
 COPY README.md /app/README.md
 COPY covid_api/ /app/covid_api/
 COPY setup.py /app/setup.py
 
-RUN pip install /app/. "mangum>=0.9.0" -t /var/task --no-binary numpy
+RUN pip install --upgrade pip
+RUN pip install . "mangum>=0.9.0" -t /var/task --no-binary numpy, pydantic
 
 # Reduce package size and remove useless files
 RUN cd /var/task && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done;
@@ -18,7 +19,5 @@ RUN find /var/task -type d -a -name 'tests' -print0 | xargs -0 rm -rf
 RUN rm -rdf /var/task/numpy/doc/
 RUN rm -rdf /var/task/stack
 
-RUN cd /var/task && zip -r9q /tmp/package.zip *
 
-COPY lambda/handler.py handler.py
-RUN zip -r9q /tmp/package.zip handler.py
+COPY lambda/handler.py /var/task/handler.py

cdk.context.json

@@ -0,0 +1,45 @@
+{
+  "vpc-provider:account=853558080719:filter.vpc-id=vpc-0fa3007e738c7bbdf:region=us-east-1:returnAsymmetricSubnets=true": {
+    "vpcId": "vpc-0fa3007e738c7bbdf",
+    "vpcCidrBlock": "10.0.0.0/16",
+    "availabilityZones": [],
+    "subnetGroups": [
+      {
+        "name": "Private",
+        "type": "Private",
+        "subnets": [
+          {
+            "subnetId": "subnet-04bc4ca3d119f6f6b",
+            "cidr": "10.0.128.0/18",
+            "availabilityZone": "us-east-1a",
+            "routeTableId": "rtb-0a01309e2f528c2bd"
+          },
+          {
+            "subnetId": "subnet-0bcd0f2d9b9ac1c56",
+            "cidr": "10.0.192.0/18",
+            "availabilityZone": "us-east-1b",
+            "routeTableId": "rtb-05251cbc837438e6c"
+          }
+        ]
+      },
+      {
+        "name": "Public",
+        "type": "Public",
+        "subnets": [
+          {
+            "subnetId": "subnet-009875640f64d198a",
+            "cidr": "10.0.0.0/18",
+            "availabilityZone": "us-east-1a",
+            "routeTableId": "rtb-0b1d4d54a9d962398"
+          },
+          {
+            "subnetId": "subnet-0e033da6876bf7014",
+            "cidr": "10.0.64.0/18",
+            "availabilityZone": "us-east-1b",
+            "routeTableId": "rtb-074500f27775c6bda"
+          }
+        ]
+      }
+    ]
+  }
+}

covid_api/api/api_v1/endpoints/detections.py

@@ -12,7 +12,7 @@
 router = APIRouter()
 
 # TODO: unhardcoded types and dates
-MLTypes = Enum("MLTypes", [(ml, ml) for ml in ["ship", "multiple", "plane"]])  # type: ignore
+MLTypes = Enum("MLTypes", [(ml, ml) for ml in ["ship", "multiple", "plane", "vehicles"]])  # type: ignore
 
 
 @router.get(

covid_api/api/utils.py

@@ -740,4 +740,6 @@ def site_date_to_scenes(site: str, date: str):
             json.loads(row["scene_id"].replace("'", '"'))
         )
 
-    return site_date_to_scenes_dict[f"{site}-{date}"]
+    # deduplicate scene list (in case multiple datasets contains the same
+    # scene id)
+    return list(set(site_date_to_scenes_dict[f"{site}-{date}"]))

covid_api/core/config.py

@@ -19,6 +19,14 @@
 MEMCACHE_PASSWORD = os.environ.get("MEMCACHE_PASSWORD")
 
 INDICATOR_BUCKET = os.environ.get("INDICATOR_BUCKET", "covid-eo-data")
+
+DATASET_METADATA_FILENAME = os.environ.get(
+    "DATASET_METADATA_FILENAME", "dev-dataset-metadata.json"
+)
+DATASET_METADATA_GENERATOR_FUNCTION_NAME = os.environ.get(
+    "DATASET_METADATA_GENERATOR_FUNCTION_NAME", "dev-dataset-metadata-generator"
+)
+
 DT_FORMAT = "%Y-%m-%d"
 MT_FORMAT = "%Y%m"
 PLANET_API_KEY = os.environ.get("PLANET_API_KEY")