Cdr metadata response

CHANGELOG.md

@@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [Unreleased]
+- Partially fixed cog_results so that it functions but doesn't include the point features.
+- Changed retrieve_cog_metadata to use cdr_response.CogMeta format
+- Updated cmaas_utils to version 2.0
+- Updated cdr_schemas to version 4.9.0
+- Changed retieve_endpoint to be member function of cdrConnector
+- Updated retrieve and validate functions to be one function, with optional parameters.
+- Updated Server.py to for new retrieve and validate calls
+- Convert code moved to cmaas_utils from this repo
+- Updated Server.py to change convert function calls to use cmaas_utils version
+- Updated CogMetadataSchema to conform with cdr_schemas v4.9.0
+- Updated FeatureResults to conform with cdr_schemas v4.9.0
+- Fixed import paths to allow for tests to be run
+- Added docstrings for cdrconnector and retrieve api functions
+### Tests
+- Updated retrieve tests 
+- Removed validate test functions
+- Removed convert tests
+- Fixed remaining tests that were failing
+- Renabled test_retrieve_cog_results
+
 ## [0.9.2] - 2024-08-22
 
 ### Added

cdrhook/cdr_endpoint_schemas.py

@@ -34,35 +34,3 @@ def pretty_str(self):
         outstr += "\n)"
         return outstr
 
-# # Returned by cog_metadata endpoint
-class GeoJsonCoord(BaseModel):
-    type: str
-    coordinates: List[List[List[float]]]
-
-class CogMetadataSchema(BaseModel):
-    citation: str
-    ngmdb_prod: str
-    scale: int
-    has_part_names: List[str]
-    ngmdb_item: int
-    cog_id: str
-    publisher: str
-    cog_url: str
-    provider_name: str
-    display_links_str: str
-    cog_size: int
-    authors: List[str]
-    provider_url: str
-    original_download_url: str
-    no_map: bool
-    thumbnail_url: str
-    state: Optional[str]
-    cog_name: str
-    publish_year: int
-    quadrangle: Optional[str]
-    alternate_name: str
-    keywords: List[str]
-    best_bounds_geojson: GeoJsonCoord
-    georeferenced_count : int
-    validated_count : int
-

cdrhook/connector.py

@@ -4,6 +4,9 @@
 from pydantic import BaseModel, Field, AnyUrl
 
 class CdrConnector(BaseModel):
+    """
+    Class to handle registration and communication with the CDR API.
+    """
     system_name : str = Field(
         description="The name of the system registering with the CDR")
     system_version : str = Field(
@@ -33,7 +36,16 @@ class CdrConnector(BaseModel):
 
     def register(self):
         """
-        Register our system to the CDR using the app_settings
+        Register our system to the CDR using the app_settings. The register call can fail if
+        another system with the same name and version is already registered, if this happens
+        you can manual deregister the other system through the CDR Docs API or change the name
+        and version of your system.
+
+        Returns:
+            str: The registration ID returned by the CDR
+
+        Raises:
+            requests.HTTPError: If the request fails
         """
         headers = {'Authorization': f'Bearer {self.token}'}
         registration = {
@@ -97,4 +109,34 @@ def __repr__(self) -> str:
 
     def __del__(self):
         if self.registration is not None:
-            self.unregister()
+            self.unregister()
+
+    def retrieve_endpoint(self, endpoint_url:str, schema:BaseModel=None, headers:dict=None):
+        """
+        Retrieve data from a CDR endpoint. If a schema is provided, the data will be converted to that schema and validated.
+
+        Args:
+            endpoint_url (str): The URL of the endpoint to retrieve data from.
+            schema (BaseModel, optional): A Pydantic schema to convert the data to. Defaults to None.
+            headers (dict, optional): A dictionary of headers to include in the request.  Defaults to None.
+
+        Returns:
+            A dictionary of the data from the endpoint or
+            An instance of the schema object if a schema was provided.
+            Data can potentionally be a list if the endpoint returns multiple items.
+
+        Raises:
+            requests.HTTPError: If the request fails
+            pydantic.ValidationError: If the data does not match the provided schema
+        """
+        if headers is None:
+            headers = {'Authorization': f'Bearer {self.token}'}
+        logging.debug(f"Retrieving {endpoint_url}")
+        r = requests.get(endpoint_url, headers=headers)
+        r.raise_for_status()
+        response = r.json()
+        if schema is not None:
+            if isinstance(response, list):
+                return [schema.model_validate(item) for item in response]
+            return schema.model_validate(response)
+        return response

cdrhook/convert.py

@@ -1,52 +1,71 @@
 from typing import List
-from cdr_endpoint_schemas import CogMetadataSchema
+from .cdr_endpoint_schemas import CogMetadataSchema
 from cdr_schemas.cdr_responses.legend_items import LegendItemResponse
 from cdr_schemas.cdr_responses.area_extractions import AreaExtractionResponse
-from cmaas_utils.types import Legend, MapUnit, MapUnitType, Layout, CMAAS_MapMetadata, Provenance
+from cmaas_utils.types import Legend, MapUnit, MapUnitType, Layout, CMAAS_MapMetadata, Provenance, AreaBounds
 
-def convert_cdr_schema_metadata_to_cmass_map_metadata(cdr_metadata:CogMetadataSchema) -> CMAAS_MapMetadata:
-    map_metadata = CMAAS_MapMetadata(provenance=Provenance(name='CDR', version='0.3.3'))
-    map_metadata.title = cdr_metadata.cog_name
-    map_metadata.authors = cdr_metadata.authors
-    map_metadata.publisher = cdr_metadata.publisher
-    map_metadata.source_url = cdr_metadata.cog_url
-    map_metadata.year = cdr_metadata.publish_year
-    map_metadata.scale = cdr_metadata.scale
-    #map_metadata.map_color =
-    #map_metadata.map_shape = 
-    #map_metadata.physiographic_region 
-    return map_metadata
+# def convert_cdr_schema_metadata_to_cmass_map_metadata(cdr_metadata:CogMetadataSchema) -> CMAAS_MapMetadata:
+#     map_metadata = CMAAS_MapMetadata(provenance=Provenance(name='CDR', version='0.3.3'))
+#     map_metadata.title = cdr_metadata.cog_name
+#     map_metadata.authors = cdr_metadata.authors
+#     map_metadata.publisher = cdr_metadata.publisher
+#     map_metadata.source_url = cdr_metadata.cog_url
+#     map_metadata.year = cdr_metadata.publish_year
+#     map_metadata.scale = cdr_metadata.scale
+#     #map_metadata.map_color =
+#     #map_metadata.map_shape = 
+#     #map_metadata.physiographic_region 
+#     return map_metadata
 
-def convert_cdr_schema_legend_items_to_cmass_legend(cdr_legend:List[LegendItemResponse]) -> Legend:
-    if len(cdr_legend) == 0:
-        return None
-    legend = Legend(provenance=Provenance(name=cdr_legend[0].system, version=cdr_legend[0].system_version))    
-    for item in cdr_legend:
-        map_unit = MapUnit(type=MapUnitType.from_str(item.category.lower()))
-        map_unit.label = item.label
-        map_unit.abbreviation = item.abbreviation
-        map_unit.description = item.description
-        map_unit.color = item.color
-        map_unit.pattern = item.pattern
-        #map_unit.overlay = 
-        map_unit.label_bbox = [item.px_bbox[0:2],item.px_bbox[2:4]]
-        legend.features.append(map_unit)
-    return legend
+# def convert_cdr_schema_legend_items_to_cmass_legend(cdr_legend:List[LegendItemResponse]) -> Legend:
+#     if len(cdr_legend) == 0:
+#         return None
+#     legend = Legend(provenance=Provenance(name=cdr_legend[0].system, version=cdr_legend[0].system_version))    
+#     for item in cdr_legend:
+#         map_unit = MapUnit(type=MapUnitType.from_str(item.category.lower()))
+#         map_unit.label = item.label
+#         map_unit.abbreviation = item.abbreviation
+#         map_unit.description = item.description
+#         map_unit.color = item.color
+#         map_unit.pattern = item.pattern
+#         #map_unit.overlay = 
+#         map_unit.label_bbox = [item.px_bbox[0:2],item.px_bbox[2:4]]
+#         legend.features.append(map_unit)
+#     return legend
 
-def convert_cdr_schema_area_extraction_to_layout(cdr_area_extraction:List[AreaExtractionResponse]) -> Layout:
-    if len(cdr_area_extraction) == 0:
-        return None
-    layout = Layout(provenance=Provenance(name=cdr_area_extraction[0].system, version=cdr_area_extraction[0].system_version))
-    for area in cdr_area_extraction:
-        if area.category == 'map_area':
-            layout.map = area.px_geojson.coordinates
-        if area.category == 'line_point_legend_area':
-            layout.line_legend = area.px_geojson.coordinates
-            layout.point_legend = area.px_geojson.coordinates
-        if area.category == 'polygon_legend_area':
-            layout.polygon_legend = area.px_geojson.coordinates
-        if area.category == 'cross_section':
-            layout.cross_section = area.px_geojson.coordinates
-        if area.category == 'correlation_diagram':
-            layout.correlation_diagram = area.px_geojson.coordinates
-    return layout
+# def convert_cdr_schema_area_extraction_to_layout(cdr_area_extraction:List[AreaExtractionResponse]) -> Layout:
+#     """
+#     Convert a list of cdr_schema AreaExtractionResponse to a cmaas_utils Layout object.
+
+#     Args:
+#         cdr_area_extraction (List[AreaExtractionResponse]): A list of cdr_schema AreaExtractionResponse objects.
+
+#     Returns:
+#         Layout: A cmaas_utils Layout object.
+#     """
+#     if len(cdr_area_extraction) == 0:
+#         return None
+#     layout = Layout(provenance=Provenance(name=cdr_area_extraction[0].system, version=cdr_area_extraction[0].system_version))
+#     for area in cdr_area_extraction:
+#         # Map area is selected by the highest confidence
+#         if area.category == 'map_area':
+#             if len(layout.map) == 0:
+#                 layout.map = [AreaBounds(geometry=area.px_geojson.coordinates, confidence=area.confidence)]
+#             else:
+#                 cur_confidence = 0
+#                 for map_area in layout.map:
+#                     if map_area.confidence is not None:
+#                         cur_confidence = max(cur_confidence, map_area.confidence)
+#                 if area.confidence > cur_confidence:
+#                     layout.map = [AreaBounds(geometry=area.px_geojson.coordinates, confidence=area.confidence)]
+#         # All other areas are concatanated to the layout
+#         if area.category == 'line_point_legend_area':
+#             layout.line_legend.append(AreaBounds(geometry=area.px_geojson.coordinates, confidence=area.confidence))
+#             layout.point_legend.append(AreaBounds(geometry=area.px_geojson.coordinates, confidence=area.confidence))
+#         if area.category == 'polygon_legend_area':
+#             layout.polygon_legend.append(AreaBounds(geometry=area.px_geojson.coordinates, confidence=area.confidence))
+#         if area.category == 'cross_section':
+#             layout.cross_section.append(AreaBounds(geometry=area.px_geojson.coordinates, confidence=area.confidence))
+#         if area.category == 'correlation_diagram':
+#             layout.correlation_diagram.append(AreaBounds(geometry=area.px_geojson.coordinates, confidence=area.confidence))
+#     return layout

cdrhook/requirements.txt

@@ -7,7 +7,7 @@ python-dotenv
 pydantic
 geopandas
 rasterio
-git+https://github.com/DARPA-CRITICALMAAS/[email protected].2
+git+https://github.com/DARPA-CRITICALMAAS/[email protected].9
 
 --extra-index-url https://test.pypi.org/simple/
-cmaas_utils==0.1.11
+cmaas_utils==0.2.0