Merge pull request #123 from NASA-IMPACT/develop

Release v0.6.0
NASA-IMPACT · May 6, 2021 · 0acd3ae · 0acd3ae
2 parents 7e13c0f + 4d31783
commit 0acd3ae
Show file tree

Hide file tree

Showing 32 changed files with 196 additions and 86 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/Dockerfiles/ecs/Dockerfile b/Dockerfiles/ecs/Dockerfile
@@ -10,6 +10,6 @@ COPY README.md /app/README.md
 COPY covid_api/ /app/covid_api/
 COPY setup.py /app/setup.py
 
-RUN pip install -e /app/. --no-cache-dir
+RUN pip install -e /app/. boto3 --no-cache-dir 
 
 CMD ["/start-reload.sh"]
diff --git a/Dockerfiles/lambda/Dockerfile b/Dockerfiles/lambda/Dockerfile
@@ -9,15 +9,23 @@ COPY covid_api/ /app/covid_api/
 COPY setup.py /app/setup.py
 
 RUN pip install --upgrade pip
-RUN pip install . "mangum>=0.9.0" -t /var/task --no-binary numpy, pydantic
+RUN pip install . "mangum==0.9.2" -t /var/task --no-binary numpy, pydantic
 
 # Reduce package size and remove useless files
 RUN cd /var/task && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done;
 RUN cd /var/task && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
 RUN cd /var/task && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
 RUN find /var/task -type d -a -name 'tests' -print0 | xargs -0 rm -rf
-RUN rm -rdf /var/task/numpy/doc/
+RUN echo "Remove lambda python packages"
+RUN rm -rdf /var/task/numpy/doc/ 
 RUN rm -rdf /var/task/stack
+RUN rm -rdf /var/task/boto3*
+RUN rm -rdf /var/task/botocore*
+RUN rm -rdf /var/task/docutils*
+RUN rm -rdf /var/task/dateutil*
+RUN rm -rdf /var/task/jmespath*
+RUN rm -rdf /var/task/s3transfer*
 
 
 COPY lambda/handler.py /var/task/handler.py
+
diff --git a/cdk.context.json b/cdk.context.json
diff --git a/covid_api/api/api_v1/endpoints/detections.py b/covid_api/api/api_v1/endpoints/detections.py
@@ -27,7 +27,6 @@ def get_detection(ml_type: MLTypes, site: SiteNames, date: str):
     """ Handle /detections requests."""
 
     try:
-
         return json.loads(
             s3_get(
                 bucket=config.INDICATOR_BUCKET,

diff --git a/covid_api/db/static/datasets/__init__.py b/covid_api/db/static/datasets/__init__.py
@@ -110,7 +110,6 @@ def get(self, spotlight_id: str, api_url: str) -> Datasets:
 
     def get_all(self, api_url: str) -> Datasets:
         """Fetch all Datasets. Overload domain with S3 scanned domain"""
-        # print(self._load_domain_metadata())
         datasets = self._process(
             datasets_domains_metadata=self._load_domain_metadata()["_all"],
             api_url=api_url,
@@ -154,6 +153,7 @@ def _process(
         (list) : datasets metadata objects (to be serialized as a pydantic Datasets
             model)
         """
+
         output_datasets = {
             k: v
             for k, v in self._data().items()
@@ -163,7 +163,7 @@ def _process(
         for k, dataset in output_datasets.items():
 
             # overload domain with domain returned from s3 file
-            dataset.domain = datasets_domains_metadata[k]["domain"]
+            dataset.domain = datasets_domains_metadata.get(k, {}).get("domain", [])
 
             # format url to contain the correct API host and
             # spotlight id (if a spotlight was requested)

diff --git a/covid_api/db/static/datasets/agriculture.json b/covid_api/db/static/datasets/agriculture.json
@@ -14,15 +14,18 @@
     },
     "exclusive_with": [
         "no2",
+        "no2-diff",
         "co2",
         "co2-diff",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "nightlights-hd",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-ship",
         "detections-plane",
         "detections-contrail",

diff --git a/covid_api/db/static/datasets/co2-diff.json b/covid_api/db/static/datasets/co2-diff.json
@@ -16,13 +16,15 @@
         "no2",
         "no2-diff",
         "co2",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "nightlights-hd",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-ship",
         "detections-plane",
         "detections-contrail",

diff --git a/covid_api/db/static/datasets/co2.json b/covid_api/db/static/datasets/co2.json
@@ -15,13 +15,15 @@
         "agriculture",
         "no2",
         "co2-diff",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "nightlights-hd",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-ship",
         "detections-plane",
         "detections-contrail",

diff --git a/covid_api/db/static/datasets/detections-contrail.json b/covid_api/db/static/datasets/detections-contrail.json
@@ -22,13 +22,15 @@
         "no2",
         "co2-diff",
         "co2",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "nightlights-hd",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-ship",
         "detections-plane",
         "detections-vehicles"

diff --git a/covid_api/db/static/datasets/detections-plane.json b/covid_api/db/static/datasets/detections-plane.json
@@ -22,13 +22,15 @@
         "no2",
         "co2-diff",
         "co2",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "nightlights-hd",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-ship",
         "detections-contrail",
         "detections-vehicles"

diff --git a/covid_api/db/static/datasets/detections-ship.json b/covid_api/db/static/datasets/detections-ship.json
@@ -22,13 +22,15 @@
         "no2",
         "co2-diff",
         "co2",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "nightlights-hd",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-plane",
         "detections-contrail",
         "detections-vehicles"

diff --git a/covid_api/db/static/datasets/detections-vehicles.json b/covid_api/db/static/datasets/detections-vehicles.json
@@ -22,13 +22,15 @@
         "no2",
         "co2-diff",
         "co2",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "nightlights-hd",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-ship",
         "detections-plane",
         "detections-contrail"

diff --git a/covid_api/db/static/datasets/fb-population-density.json b/covid_api/db/static/datasets/fb-population-density.json
@@ -0,0 +1,50 @@
+{
+    "id": "fb-population-density",
+    "name": "Facebook Population Density",
+    "type": "raster",
+    "time_unit": "day",
+    "source": {
+        "type": "raster",
+        "tiles": [
+            "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/dataforgood-fb-population-density/cog.tif&rescale=0,69&resampling_method=nearest&color_map=ylorrd"
+        ]
+    },
+    "paint": {
+        "raster-opacity": 0.9
+    },
+    "exclusive_with": [
+        "agriculture",
+        "no2",
+        "co2-diff",
+        "gibs-population",
+        "car-count",
+        "nightlights-viirs",
+        "nightlights-hd",
+        "detection-multi",
+        "water-chlorophyll",
+        "water-spm",
+        "water-pzd",
+        "detections-ship",
+        "detections-plane",
+        "detections-vehicles"
+    ],
+    "enabled": true,
+    "swatch": {
+        "color": "#C0C0C0",
+        "name": "Grey"
+    },
+    "legend": {
+        "type": "gradient",
+        "min": "less",
+        "max": "more",
+        "stops": [
+            "#99c5e0",
+            "#f9eaa9",
+            "#f7765d",
+            "#c13b72",
+            "#461070",
+            "#050308"
+        ]
+    },
+    "info": "Facebook high-resolution population density: Darker areas indicate higher population density areas and lighter areas indicate lower population density areas"
+}
diff --git a/covid_api/db/static/datasets/population.json → ...i/db/static/datasets/gibs-population.json b/covid_api/db/static/datasets/population.json → ...i/db/static/datasets/gibs-population.json
@@ -1,5 +1,5 @@
 {
-  "id": "population",
+  "id": "gibs-population",
   "name": "Population",
   "type": "raster",
   "time_unit": "day",
@@ -21,7 +21,8 @@
     "detection-ship",
     "detection-multi",
     "water-chlorophyll",
-    "water-spm"
+    "water-spm",
+    "water-pzd"
   ],
   "swatch": {
     "color": "#C0C0C0",

diff --git a/covid_api/db/static/datasets/nightlights-hd.json b/covid_api/db/static/datasets/nightlights-hd.json
@@ -8,20 +8,22 @@
     "source": {
         "type": "raster",
         "tiles": [
-            "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/bmhd_30m_monthly/BMHD_VNP46A2_{spotlightId}_{date}_cog.tif&resampling_method=bilinear&bidx=1%2C2%2C3"
+            "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/bmhd_30m_monthly/BMHD_VNP46A2_{spotlightId}_{date}_cog.tif&bidx=1&rescale=0,255&color_map=inferno"
         ]
     },
     "exclusive_with": [
         "agriculture",
         "no2",
         "co2-diff",
         "co2",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-ship",
         "detections-plane",
         "detections-contrail",

diff --git a/covid_api/db/static/datasets/nightlights-viirs.json b/covid_api/db/static/datasets/nightlights-viirs.json
@@ -16,12 +16,14 @@
         "no2",
         "co2-diff",
         "co2",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-hd",
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "detections-ship",
         "detections-plane",
         "detections-contrail",

diff --git a/covid_api/db/static/datasets/no2-diff.json b/covid_api/db/static/datasets/no2-diff.json
@@ -17,6 +17,7 @@
     "exclusive_with": [
         "co2",
         "co2-diff",
+        "fb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
@@ -25,6 +26,7 @@
         "detection-multi",
         "water-chlorophyll",
         "water-spm",
+        "water-pzd",
         "no2"
     ],
     "enabled": false,

diff --git a/covid_api/db/static/datasets/no2.json b/covid_api/db/static/datasets/no2.json
@@ -18,13 +18,15 @@
     "agriculture",
     "co2",
     "co2-diff",
+    "fb-population-density",
     "gibs-population",
     "car-count",
     "nightlights-viirs",
     "nightlights-hd",
     "detection-multi",
     "water-chlorophyll",
     "water-spm",
+    "water-pzd",
     "detections-ship",
     "detections-plane",
     "detections-contrail",

diff --git a/covid_api/db/static/datasets/recovery.json b/covid_api/db/static/datasets/recovery.json
@@ -17,14 +17,16 @@
         "agriculture",
         "co2",
         "co2-diff",
+        "gifb-population-density",
         "gibs-population",
         "car-count",
         "nightlights-viirs",
         "nightlights-hd",
         "detection-ship",
         "detection-multi",
         "water-chlorophyll",
-        "water-spm"
+        "water-spm",
+        "water-pzd"
     ],
     "enabled": true,
     "swatch": {
-Original file line number
+Diff line change
@@ Expand Up @@
         """ Handle /detections requests."""
         try:
             return json.loads(
                 s3_get(
                     bucket=config.INDICATOR_BUCKET,
@@ Expand Down @@