Merge branch 'simplify_plugins' of https://github.com/cedadev/stac-ge…

…nerator into simplify_plugins Conflicts: requirements.txt stac_generator/plugins/extraction_methods/asset_aggregator.py stac_generator/plugins/extraction_methods/ceda_vocabulary.py stac_generator/plugins/extraction_methods/datetime_bound_to_centroid.py stac_generator/plugins/extraction_methods/dot_seperated_str.py stac_generator/plugins/extraction_methods/elasticsearch_aggregation.py stac_generator/plugins/extraction_methods/elasticsearch_assets.py stac_generator/plugins/extraction_methods/general_function.py stac_generator/plugins/extraction_methods/geometry_to_bbox.py stac_generator/plugins/extraction_methods/header/header.py stac_generator/plugins/extraction_methods/iso_date.py stac_generator/plugins/extraction_methods/json_file.py stac_generator/plugins/extraction_methods/lambda.py stac_generator/plugins/extraction_methods/netcdf.py stac_generator/plugins/extraction_methods/open_zip.py stac_generator/plugins/extraction_methods/regex_assets.py stac_generator/plugins/extraction_methods/remove.py stac_generator/plugins/extraction_methods/stac_bbox.py stac_generator/plugins/extraction_methods/xml.py stac_generator/plugins/mappings/stac.py stac_generator/plugins/outputs/stac_fastapi.py
cedadev · Feb 5, 2024 · 92af0b8 · 92af0b8
2 parents c628c13 + faebe8f
commit 92af0b8
Show file tree

Hide file tree

Showing 16 changed files with 78 additions and 61 deletions.
diff --git a/docs_requirements.txt b/docs_requirements.txt
@@ -5,7 +5,7 @@ git+https://github.com/cedadev/stac-generator.git#egg=stac_generator
 Babel==2.9.1
 boto3==1.18.53
 botocore==1.21.53
-certifi==2022.12.7
+certifi==2023.7.22
 charset-normalizer==2.0.6
 cloudpickle==2.0.0
 dask==2021.10.0
@@ -25,13 +25,13 @@ packaging==21.0
 partd==1.2.0
 pika==1.2.0
 pydantic==1.8.2
-Pygments==2.10.0
+Pygments==2.15.0
 pyparsing==2.4.7
 python-dateutil==2.8.2
 python-magic==0.4.24
 pytz==2021.3
 PyYAML==5.4.1
-requests==2.26.0
+requests==2.31.0
 s3transfer==0.5.0
 siphon==0.9
 six==1.16.0
@@ -49,4 +49,4 @@ sphinxcontrib-serializinghtml==1.1.5
 toolz==0.11.1
 tqdm==4.62.3
 typing-extensions==4.0.0
-urllib3==1.26.7
+urllib3==1.26.18
diff --git a/requirements.txt b/requirements.txt
@@ -1,13 +1,12 @@
-aiohttp==3.7.4.post0
+aiohttp==3.9.0
 alabaster==0.7.12
 anytree==2.8.0
 async-timeout==3.0.1
 Babel==2.9.1
 boto3==1.18.44
 botocore==1.21.44
 cachetools==4.2.4
-certifi==2022.12.7
-ceda-directory-tree==1.0.2
+certifi==2023.7.22
 cf_xarray==0.8.4
 chardet==4.0.0
 click==8.1.3
@@ -23,19 +22,20 @@ multidict==5.1.0
 packaging==20.9
 pika==1.2.0
 pydantic==2.1.1
-Pygments==2.9.0
+Pygments==2.15.0
 pyparsing==2.4.7
 python-dateutil==2.8.2
 python-magic==0.4.25
 pytz==2021.1
 PyYAML==5.4.1
-requests==2.25.1
+requests==2.31.0
 s3transfer==0.5.0
 siphon==0.9
 six==1.16.0
 snowballstemmer==2.1.0
 tqdm==4.62.3
 typing-extensions==4.7.1
-urllib3==1.26.5
+urllib3==1.26.18
 xarray==2023.9.0
 git+https://github.com/cedadev/extraction-methods
+yarl==1.6.3
diff --git a/stac_generator/core/baker.py b/stac_generator/core/baker.py
@@ -17,7 +17,6 @@
 import logging
 
 # Python imports
-from collections import defaultdict
 from functools import lru_cache
 from pathlib import Path
 from typing import Optional
@@ -145,7 +144,9 @@ def load_recipe(self, key: str, stac_type: str) -> Recipe:
         """
         recipe = self.recipes[stac_type][key]
 
-        recipe.member_of = [self.recipes[link["type"]][link["key"]] for link in recipe.links]
+        recipe.member_of = [
+            self.recipes[link["type"]][link["key"]] for link in recipe.links
+        ]
 
         return recipe
 
@@ -167,4 +168,4 @@ def get(self, path: str, stac_type: str) -> Recipe:
         raise ValueError(f"No Recipe found for path: {path}")
 
     def get_maps(self):
-        return self.paths_map, self.location_map
+        return self.paths_map, self.location_map
diff --git a/stac_generator/core/generator.py b/stac_generator/core/generator.py
@@ -84,7 +84,11 @@ def _load_extraction_method(
 
             for extraction_method in inputs.get("extraction_methods", []):
                 if isinstance(extraction_method, dict):
-                    extraction_methods.append(self._load_extraction_method(ExtractionMethodConf(**extraction_method), **kwargs))
+                    extraction_methods.append(
+                        self._load_extraction_method(
+                            ExtractionMethodConf(**extraction_method), **kwargs
+                        )
+                    )
 
                 else:
                     extraction_methods.append(extraction_method)

diff --git a/stac_generator/core/utils.py b/stac_generator/core/utils.py
@@ -13,18 +13,14 @@
 import re
 
 # Python imports
-from datetime import datetime
 from pathlib import Path
 from string import Template
 
 # Typing imports
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Union
 
 import yaml
 
-# Third party imports
-from dateutil.parser import parse
-
 from stac_generator.core.exceptions import NoPluginsError
 from stac_generator.core.handler_picker import HandlerPicker
 

diff --git a/stac_generator/plugins/generators/asset.py b/stac_generator/plugins/generators/asset.py
@@ -36,7 +36,9 @@ def _process(self, body: dict, **kwargs) -> None:
 
         :return:
         """
-        recipe = self.recipes.get(kwargs.get("recipe_path", body["uri"]), self.TYPE.value)
+        recipe = self.recipes.get(
+            kwargs.get("recipe_path", body["uri"]), self.TYPE.value
+        )
 
         LOGGER.debug(
             "Generating %s : %s with recipe %s", self.TYPE.value, body["uri"], recipe

diff --git a/stac_generator/plugins/generators/collection.py b/stac_generator/plugins/generators/collection.py
@@ -42,7 +42,9 @@ def _process(self, body: dict, **kwargs) -> None:
 
         :return:
         """
-        recipe = self.recipes.get(kwargs.get("recipe_path", body["uri"]), self.TYPE.value)
+        recipe = self.recipes.get(
+            kwargs.get("recipe_path", body["uri"]), self.TYPE.value
+        )
 
         LOGGER.debug(
             "Generating %s : %s with recipe %s", self.TYPE.value, body["uri"], recipe

diff --git a/stac_generator/plugins/generators/item.py b/stac_generator/plugins/generators/item.py
@@ -42,7 +42,9 @@ def _process(self, body: dict, **kwargs) -> None:
 
         :return:
         """
-        recipe = self.recipes.get(kwargs.get("recipe_path", body["uri"]), self.TYPE.value)
+        recipe = self.recipes.get(
+            kwargs.get("recipe_path", body["uri"]), self.TYPE.value
+        )
 
         LOGGER.debug(
             "Generating %s : %s with recipe %s", self.TYPE.value, body["uri"], recipe

diff --git a/stac_generator/plugins/inputs/text_file.py b/stac_generator/plugins/inputs/text_file.py
@@ -55,7 +55,6 @@ def __init__(self, **kwargs):
             self.file_list = [self.filepath]
 
     def run(self, generator: BaseGenerator):
-
         start = datetime.now()
         total_generated = 0
         unique_lines = set()

diff --git a/stac_generator/plugins/mappings/jinja2.py b/stac_generator/plugins/mappings/jinja2.py
@@ -6,7 +6,6 @@
 __contact__ = "[email protected]"
 
 import logging
-from datetime import datetime
 
 from jinja2 import Environment, FileSystemLoader
 

diff --git a/stac_generator/plugins/mappings/stac.py b/stac_generator/plugins/mappings/stac.py
@@ -50,7 +50,6 @@ def item(self, body:dict) -> dict:
             },
         }
 
-        extent = {}
         if "datetime" in body:
             output["properties"]["datetime"] = self.datetime_field(body, "datetime")
 
@@ -92,13 +91,12 @@ def collection(self, body: dict) -> dict:
                 },
                 "spatial": {
                     "bbox": None,
-                }
+                },
             },
             "summaries": {},
             "assets": {},
         }
 
-        extent = {}
         if "interval" in body:
             output["extent"]["temporal"]["interval"] = body.pop("interval")
 
@@ -112,17 +110,16 @@ def collection(self, body: dict) -> dict:
 
         return output
 
-
     def run(
         self,
         body: dict,
         recipe: Recipe,
         **kwargs,
     ) -> dict:
-        if kwargs['TYPE'].value == "item":
+        if kwargs["TYPE"].value == "item":
             return self.item(body)
 
-        elif kwargs['TYPE'].value == "collection":
+        elif kwargs["TYPE"].value == "collection":
             return self.collection(body)
 
-        return body
+        return body
diff --git a/stac_generator/plugins/outputs/elasticsearch.py b/stac_generator/plugins/outputs/elasticsearch.py
@@ -40,9 +40,6 @@
 __license__ = "BSD - see LICENSE file in top-level package directory"
 __contact__ = "[email protected]"
 
-from datetime import datetime
-from typing import Dict
-
 from elasticsearch import Elasticsearch
 
 from stac_generator.core.output import BaseOutput
@@ -74,7 +71,7 @@ def __init__(self, **kwargs):
                 self.es.indices.create(self.index_name, body=mapping)
 
     @staticmethod
-    def _format_bbox(data: Dict) -> Dict:
+    def _format_bbox(data: dict) -> dict:
         """
         Convert WGS84 coordinates into GeoJSON and
         format for Elasticsearch. Replaces the bbox key.
@@ -96,7 +93,7 @@ def _format_bbox(data: Dict) -> Dict:
         return data
 
     @staticmethod
-    def _format_temporal_extent(data: Dict) -> Dict:
+    def _format_temporal_extent(data: dict) -> dict:
         """
         Convert `extent object<https://github.com/radiantearth/stac-spec/blob/master/collection-spec/collection-spec.md#extent-object>_` for Elasticsearch.
 
@@ -116,7 +113,7 @@ def _format_temporal_extent(data: Dict) -> Dict:
 
         return data
 
-    def clean(self, data: Dict) -> Dict:
+    def clean(self, data: dict) -> dict:
         """
         Condition the input dictionary for elasticsearch
         :param data: Input dictionary
@@ -129,7 +126,7 @@ def clean(self, data: Dict) -> Dict:
 
         return data
 
-    def _remove_old(self, data: str, previous_ids: dict) -> Dict:
+    def _remove_old(self, data: str, previous_ids: dict) -> dict:
         """
         Condition the input dictionary for elasticsearch
         :param data: Input dictionary
@@ -142,7 +139,6 @@ def _remove_old(self, data: str, previous_ids: dict) -> Dict:
             self.es.delete(index=self.index_name, id=previous_id)
 
     def export(self, data: dict, **kwargs) -> None:
-
         data = self.clean(data)
 
         if self.remove_old:

diff --git a/stac_generator/plugins/outputs/json_file.py b/stac_generator/plugins/outputs/json_file.py
@@ -21,7 +21,6 @@
     * - ``filename_term``
       - ``str``
       - ``REQUIRED`` Term to be used for the file name (typically the id).
-      
 
 Example Configuration:
     .. code-block:: yaml

diff --git a/stac_generator/plugins/outputs/stac_fastapi.py b/stac_generator/plugins/outputs/stac_fastapi.py
@@ -40,14 +40,17 @@
 __license__ = "BSD - see LICENSE file in top-level package directory"
 __contact__ = "[email protected]"
 
-import requests
+import logging
 from urllib.parse import urljoin
 import logging
 
+import requests
+
 from stac_generator.core.output import BaseOutput
 
 LOGGER = logging.getLogger(__name__)
 
+
 class STACFastAPIOutput(BaseOutput):
     """
     Connects to an elasticsearch instance and exports the
@@ -58,7 +61,7 @@ class STACFastAPIOutput(BaseOutput):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         if not hasattr(self, "verify"):
-          self.verify = True
+            self.verify = True
 
     def item(self, data: dict) -> None:
         collections = data["collection"]
@@ -67,28 +70,36 @@ def item(self, data: dict) -> None:
             collections = [collections]
 
         for collection in collections:
-
             collection = data["collection"] = collection.lower()
 
             response = requests.post(
-                urljoin(self.api_url, f"collections/{collection}/items"), json=data, verify=self.verify
+                urljoin(self.api_url, f"collections/{collection}/items"),
+                json=data,
+                verify=self.verify,
             )
 
             if response.status_code == 404:
                 response_json = response.json()
 
-                if response_json["description"] == f"Collection {collection} does not exist":
+                if (
+                    response_json["description"]
+                    == f"Collection {collection} does not exist"
+                ):
                     collection = {
                         "type": "Collection",
                         "id": collection,
                     }
 
                     response = requests.post(
-                        urljoin(self.api_url, "collections"), json=collection, verify=self.verify
+                        urljoin(self.api_url, "collections"),
+                        json=collection,
+                        verify=self.verify,
                     )
 
                     response = requests.post(
-                        urljoin(self.api_url, f"collections/{collection}/items"), json=data, verify=self.verify
+                        urljoin(self.api_url, f"collections/{collection}/items"),
+                        json=data,
+                        verify=self.verify,
                     )
 
             if response.status_code == 409:
@@ -103,18 +114,32 @@ def item(self, data: dict) -> None:
                         LOGGER.warning(f"FastAPI Output Update failed with status code: {response.status_code} and response text: {response.text}")
 
             elif response.status_code != 200:
-                LOGGER.warning(f"FastAPI Output failed with status code: {response.status_code} and response text: {response.text}")
+                LOGGER.warning(
+                    "FastAPI Output failed to post to STAC Fastapi items endpoint returned status code: %s and response text: %s request data: %s",
+                    response.status_code,
+                    response.text,
+                    data,
+                )
 
     def collection(self, data: dict) -> None:
-
         response = requests.post(
-            urljoin(self.api_url, "collections"), json=data, verify=self.verify
+            urljoin(self.api_url, "collections"),
+            json=data,
+            verify=self.verify,
+            timeout=180,
         )
 
-    def export(self, data: dict, **kwargs) -> None:
+        if response.status_code != 200:
+            LOGGER.warning(
+                "FastAPI Output failed to post to STAC Fastapi collections endpoint returned status code: %s and response text: %s request data: %s",
+                response.status_code,
+                response.text,
+                data,
+            )
 
-        if kwargs['TYPE'].value == "item":
+    def export(self, data: dict, **kwargs) -> None:
+        if kwargs["TYPE"].value == "item":
             self.item(data)
 
-        elif kwargs['TYPE'].value == "collection":
+        elif kwargs["TYPE"].value == "collection":
             self.collection(data)