Merge pull request #9 from DARPA-CRITICALMAAS/cdrhook_server_update

Cdrhook server update

.github/workflows/docker.yml

@@ -24,7 +24,8 @@ jobs:
         include:
           - name: cdrhook
             FOLDER: cdrhook
-            PLATFORM: "linux/amd64,linux/arm64"
+            #PLATFORM: "linux/amd64,linux/arm64"
+            PLATFORM: "linux/amd64"
             IMAGE: criticalmaas-cdr
           - name: uploader
             FOLDER: uploader

.github/workflows/pytest.yml

@@ -17,6 +17,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.10"]
     steps:
@@ -29,7 +30,10 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f cdrhook/requirements.txt ]; then pip install -r cdrhook/requirements.txt; fi
+    - name: Setup Env
+      run: |
+        echo "CDR_TOKEN=${{ secrets.CDR_TOKEN }}" >> $GITHUB_ENV
     - name: Test with pytest
       run: |
         pytest tests

.gitignore

@@ -3,3 +3,8 @@
 venv
 __pycache__
 docker-compose.override.yml
+
+*.pyc
+*.DS_Store
+[Ll]ogs
+data

CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.8.0] - 2024-08-06
+
+### Added 
+- Added connection and retrieve api for CDR interface
+- Added support for downloading legends from the CDR
+- Added unittests for cdrhook process_cog code
+- Added pytest github action
+- Added linting github action
+- File `systems.json` controls order to check for map_area and polygon_legent
+
+### Changed
+- Updated cdrhook server code
+- Updated message interface for download queue "map_area" -> "map_data"
+- cdrhook has default log level of INFO (can be changed with LOGLEVEL environment variable)
+
+
 ## [0.7.3] - 2024-05-13
 
 ### Added

cdrhook/Dockerfile

@@ -14,10 +14,18 @@ ENV PYTHONUNBUFFERED=1 \
     CALLBACK_USERNAME="" \
     CALLBACK_PASSWORD="" \
     RABBITMQ_URI="amqp://guest:guest@localhost:5672/%2F" \
+    LOGLEVEL="INFO" \
     PREFIX=""
 
 VOLUME /data
 
+# setup packages needed
+#    apt-get -y install python3-gdal libgdal-dev libgl1 && \
+RUN apt-get update && \
+    apt-get -y install python3-gdal libgdal-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 COPY requirements.txt ./
 RUN pip install -r ./requirements.txt
 

cdrhook/cdr_endpoint_schemas.py

@@ -0,0 +1,68 @@
+from typing import List, Optional
+from pydantic import BaseModel, Field
+
+# # Returned by cog_download enpoint
+class CogDownloadSchema(BaseModel):
+    """
+    The response schema from the CDR cog download endpoint.
+    """
+    cog_url: str = Field(description="The URL to download the geotif of the requested cog")
+    ngmdb_prod: Optional[str] = Field(description="???")
+    ngmdb_item: Optional[int] = Field(description="???")
+
+# # Returned by cog_system_versions endpoint
+class SystemId(BaseModel):
+    """
+    The system ID tuple used by the CDR to identify the provence of a peice of data.
+    System versions endpoint returns a list of these.
+    """
+    name: str = Field(description="The name of the system")
+    version: str = Field(description="The version of the system")
+
+class CogSystemVersionsSchema(BaseModel):
+    """
+    The response schema from the CDR cog system versions endpoint.
+    """
+    system_versions: List[SystemId]
+
+    def pretty_str(self):
+        """
+        Return a pretty string representation of the system versions.
+        """
+        outstr = "CogSystemVersionsSchema(\n"
+        outstr += "\n".join([f"\t{s.name} - {s.version}," for s in self.system_versions])[:-1]
+        outstr += "\n)"
+        return outstr
+
+# # Returned by cog_metadata endpoint
+class GeoJsonCoord(BaseModel):
+    type: str
+    coordinates: List[List[List[float]]]
+
+class CogMetadataSchema(BaseModel):
+    citation: str
+    ngmdb_prod: str
+    scale: int
+    has_part_names: List[str]
+    ngmdb_item: int
+    cog_id: str
+    publisher: str
+    cog_url: str
+    provider_name: str
+    display_links_str: str
+    cog_size: int
+    authors: List[str]
+    provider_url: str
+    original_download_url: str
+    no_map: bool
+    thumbnail_url: str
+    state: Optional[str]
+    cog_name: str
+    publish_year: int
+    quadrangle: Optional[str]
+    alternate_name: str
+    keywords: List[str]
+    best_bounds_geojson: GeoJsonCoord
+    georeferenced_count : int
+    validated_count : int
+

cdrhook/connector.py

@@ -0,0 +1,100 @@
+import logging
+import requests
+from typing import List, Optional
+from pydantic import BaseModel, Field, AnyUrl
+
+class CdrConnector(BaseModel):
+    system_name : str = Field(
+        description="The name of the system registering with the CDR")
+    system_version : str = Field(
+        description="The version of the system registering with the CDR")
+    token : str = Field(
+        description="The token used to authenticate with the CDR")
+    callback_url : AnyUrl = Field(
+        description="The URL to which the CDR will send callbacks")
+    callback_secret : str = Field(
+        default="",
+        description="The secret to use for the webhook")
+    callback_username : str = Field(
+        default="",
+        description="The username to use for the webhook")
+    callback_password : str = Field(
+        default="",
+        description="The password to use for the webhook")
+    events : List[str] = Field(
+        default_factory=list,
+        description="The events to register for, leaving blank will register for all events")
+    cdr_url : AnyUrl = Field(
+        default="https://api.cdr.land",
+        description="The URL of the CDR API")
+    registration : Optional[str] = Field(
+        default=None,
+        description="The registration ID returned by the CDR")
+
+    def register(self):
+        """
+        Register our system to the CDR using the app_settings
+        """
+        headers = {'Authorization': f'Bearer {self.token}'}
+        registration = {
+            "name": self.system_name,
+            "version": self.system_version,
+            "callback_url": str(self.callback_url),
+            "webhook_secret": self.callback_secret,
+            "auth_header": self.callback_username,
+            "auth_token": self.callback_password,
+            "events": self.events
+        }
+        logging.info(f"Registering with CDR: [system_name : {registration['name']}, system_version : {registration['version']}, callback_url : {registration['callback_url']}")
+        r = requests.post(f"{self.cdr_url}/user/me/register", json=registration, headers=headers)
+        logging.debug(r.text)
+        r.raise_for_status()
+        self.registration = r.json()["id"]
+        logging.info(f"Registered with CDR, registration id : {self.registration}")
+        return r.json()["id"]
+
+    def unregister(self):
+        """
+        Unregister our system from the CDR
+        """
+        # unregister from the CDR
+        headers = {'Authorization': f"Bearer {self.token}"}
+        logging.info("Unregistering with CDR")
+        r = requests.delete(f"{self.cdr_url}/user/me/register/{self.registration}", headers=headers)
+        logging.info("Unregistered with CDR")
+        r.raise_for_status()
+        self.registration = None
+
+    def __str__(self) -> str:
+        repr = "CdrConnector("
+        repr += f"system_name='{self.system_name}', "
+        repr += f"system_version='{self.system_version}', "
+        repr += f"token='{self.token[:8]}...', "
+        repr += f"callback_url='{self.callback_url}', "
+        repr += f"callback_secret='{self.callback_secret[:8]}...', "
+        repr += f"callback_username='{self.callback_username}', "
+        repr += "callback_password='...', "
+        repr += f"events={self.events}, "
+        repr += f"cdr_url='{self.cdr_url}', "
+        repr += f"registration={self.registration[:8]}..."
+        repr += ")"
+        return repr
+
+    def __repr__(self) -> str:
+        repr = "CdrConnector("
+        repr += f"system_name='{self.system_name}', "
+        repr += f"system_version='{self.system_version}', "
+        repr += f"token='{self.token[:8]}...', "
+        repr += f"callback_url='{self.callback_url}', "
+        repr += f"callback_secret='{self.callback_secret[:8]}...', "
+        repr += f"callback_username='{self.callback_username}', "
+        repr += "callback_password='...', "
+        repr += f"events={self.events}, "
+        repr += f"cdr_url='{self.cdr_url}', "
+        repr += f"registration={self.registration[:8]}..."
+        repr += ")"
+        return repr
+
+    def __del__(self):
+        if self.registration is not None:
+            self.unregister()

cdrhook/convert.py

@@ -0,0 +1,48 @@
+from typing import List
+from cdr_endpoint_schemas import CogMetadataSchema
+from cdr_schemas.cdr_responses.legend_items import LegendItemResponse
+from cdr_schemas.cdr_responses.area_extractions import AreaExtractionResponse
+from cmaas_utils.types import Legend, MapUnit, MapUnitType, Layout, CMAAS_MapMetadata, Provenance
+
+def convert_cdr_schema_metadata_to_cmass_map_metadata(cdr_metadata:CogMetadataSchema) -> CMAAS_MapMetadata:
+    map_metadata = CMAAS_MapMetadata(provenance=Provenance(name='CDR', version='0.3.3'))
+    map_metadata.title = cdr_metadata.cog_name
+    map_metadata.authors = cdr_metadata.authors
+    map_metadata.publisher = cdr_metadata.publisher
+    map_metadata.source_url = cdr_metadata.cog_url
+    map_metadata.year = cdr_metadata.publish_year
+    map_metadata.scale = cdr_metadata.scale
+    #map_metadata.map_color =
+    #map_metadata.map_shape = 
+    #map_metadata.physiographic_region 
+    return map_metadata
+
+def convert_cdr_schema_legend_items_to_cmass_legend(cdr_legend:List[LegendItemResponse]) -> Legend:
+    legend = Legend(provenance=Provenance(name=cdr_legend[0].system, version=cdr_legend[0].system_version))    
+    for item in cdr_legend:
+        map_unit = MapUnit(type=MapUnitType.from_str(item.category.lower()))
+        map_unit.label = item.label
+        map_unit.abbreviation = item.abbreviation
+        map_unit.description = item.description
+        map_unit.color = item.color
+        map_unit.pattern = item.pattern
+        #map_unit.overlay = 
+        map_unit.bounding_box = [item.px_bbox[0:2],item.px_bbox[2:4]]
+        legend.features.append(map_unit)
+    return legend
+
+def convert_cdr_schema_area_extraction_to_layout(cdr_area_extraction:List[AreaExtractionResponse]) -> Layout:
+    layout = Layout(provenance=Provenance(name=cdr_area_extraction[0].system, version=cdr_area_extraction[0].system_version))
+    for area in cdr_area_extraction:
+        if area.category == 'map_area':
+            layout.map = area.px_geojson.coordinates
+        if area.category == 'line_point_legend_area':
+            layout.line_legend = area.px_geojson.coordinates
+            layout.point_legend = area.px_geojson.coordinates
+        if area.category == 'polygon_legend_area':
+            layout.polygon_legend = area.px_geojson.coordinates
+        if area.category == 'cross_section':
+            layout.cross_section = area.px_geojson.coordinates
+        if area.category == 'correlation_diagram':
+            layout.correlation_diagram = area.px_geojson.coordinates
+    return layout

cdrhook/models.json

@@ -1,5 +1,3 @@
 {
-    "golden_muscat": ["map_area", "polygon_legend_area"],
-    "flat_iceberg": ["map_area", "line_point_legend"],
-    "drab_volcano": ["map_area"]
+    "golden_muscat": ["map_area", "polygon_legend_area"]
 }

cdrhook/requirements.txt

@@ -3,3 +3,11 @@ waitress
 flask_httpauth
 requests
 pika
+python-dotenv 
+pydantic
+geopandas
+rasterio
+git+https://github.com/DARPA-CRITICALMAAS/[email protected]
+
+--extra-index-url https://test.pypi.org/simple/
+cmaas_utils>=0.1.10