From cbf930ab01339ff3c65b564c34b43e7f8f1a04ae Mon Sep 17 00:00:00 2001 From: Vincent Privat Date: Thu, 24 Oct 2024 16:23:49 +0200 Subject: [PATCH] RSPY-491 - fix CADIP Session OData queries --- services/cadip/config/cadip_ws_config.yaml | 138 +++++++++++----- .../config/cadip_ws_config_token_module.yaml | 149 +++++++++++++----- .../cadip/rs_server_cadip/api/cadip_search.py | 2 + services/cadip/rs_server_cadip/cadip_utils.py | 8 +- .../data_retrieval/eodag_provider.py | 44 +++--- 5 files changed, 249 insertions(+), 92 deletions(-) diff --git a/services/cadip/config/cadip_ws_config.yaml b/services/cadip/config/cadip_ws_config.yaml index 13b66220d..1c1c5e94c 100644 --- a/services/cadip/config/cadip_ws_config.yaml +++ b/services/cadip/config/cadip_ws_config.yaml @@ -90,10 +90,10 @@ cadip: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -189,10 +189,10 @@ ins: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -288,10 +288,10 @@ mps: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -387,10 +387,10 @@ mti: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -486,10 +486,10 @@ nsg: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -585,10 +585,10 @@ sgs: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -632,7 +632,7 @@ cadip_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5000/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -719,13 +719,13 @@ cadip_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -734,6 +734,17 @@ cadip_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5000/Sessions" @@ -767,7 +778,7 @@ ins_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5000/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -854,13 +865,13 @@ ins_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -869,6 +880,17 @@ ins_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5100/Sessions" @@ -902,7 +924,7 @@ mps_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5101/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -989,13 +1011,13 @@ mps_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -1004,6 +1026,17 @@ mps_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5101/Sessions" @@ -1037,7 +1070,7 @@ mti_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5102/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -1124,13 +1157,13 @@ mti_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -1139,6 +1172,17 @@ mti_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5102/Sessions" @@ -1172,7 +1216,7 @@ nsg_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5103/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -1259,13 +1303,13 @@ nsg_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -1274,6 +1318,17 @@ nsg_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5103/Sessions" @@ -1307,7 +1362,7 @@ sgs_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5104/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -1394,13 +1449,13 @@ sgs_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -1409,6 +1464,17 @@ sgs_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5104/Sessions" diff --git a/services/cadip/config/cadip_ws_config_token_module.yaml b/services/cadip/config/cadip_ws_config_token_module.yaml index 68bdd79d0..34424d481 100644 --- a/services/cadip/config/cadip_ws_config_token_module.yaml +++ b/services/cadip/config/cadip_ws_config_token_module.yaml @@ -80,10 +80,10 @@ cadip: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -168,10 +168,10 @@ ins: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -256,10 +256,10 @@ mps: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -344,10 +344,10 @@ mti: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -432,10 +432,10 @@ nsg: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -520,10 +520,10 @@ sgs: free_text_search_operations: $filter: union: ' OR ' - wrapper: '"{}"' + wrapper: '{}' operations: and: - - 'SessionID in {SessionIds}' + - 'SessionID in ({SessionIds})' - 'SessionID eq {SessionID}' - 'PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}' - 'PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}' @@ -533,6 +533,17 @@ sgs: next_page_url_tpl: '{url}?{search}&$top={items_per_page}' total_items_nb_key_path: '$.total' + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: 'http://127.0.0.1:5104/Files' @@ -556,7 +567,7 @@ cadip_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5000/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -643,13 +654,13 @@ cadip_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -658,6 +669,17 @@ cadip_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5000/Sessions" @@ -680,7 +702,7 @@ ins_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5100/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -767,13 +789,13 @@ ins_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -782,6 +804,17 @@ ins_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5100/Sessions" @@ -804,7 +837,7 @@ mps_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5101/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -891,13 +924,13 @@ mps_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -906,6 +939,17 @@ mps_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5101/Sessions" @@ -928,7 +972,7 @@ mti_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5102/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -1015,13 +1059,13 @@ mti_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -1030,6 +1074,17 @@ mti_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5102/Sessions" @@ -1052,7 +1107,7 @@ nsg_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5103/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -1139,13 +1194,13 @@ nsg_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -1154,6 +1209,17 @@ nsg_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5103/Sessions" @@ -1176,7 +1242,7 @@ sgs_session: type: ODataV4Search api_endpoint: "http://127.0.0.1:5104/Sessions" need_auth: true - results_entry: responses | $ + results_entry: $.value metadata_mapping: id: - null @@ -1263,13 +1329,13 @@ sgs_session: free_text_search_operations: $filter: union: " OR " - wrapper: '"{}"' + wrapper: '{}' operations: and: - - "SessionId in {SessionIds}" + - "SessionId in ({SessionIds})" - "SessionId eq {SessionId}" - "Satellite eq {platform}" - - "Satellite in {platforms}" + - "Satellite in ({platforms})" - "PublicationDate gt {startTimeFromAscendingNode#to_iso_utc_datetime}" - "PublicationDate lt {completionTimeFromAscendingNode#to_iso_utc_datetime}" @@ -1278,6 +1344,17 @@ sgs_session: next_page_url_tpl: "{url}?{search}&$top={items_per_page}&$expand=Files" total_items_nb_key_path: "$.total" + sort: + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: DownlinkStart + completionTimeFromAscendingNode: DownlinkStop + publicationDate: PublicationDate + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + download: type: HTTPDownload base_uri: "https://127.0.0.1:5104/Sessions" diff --git a/services/cadip/rs_server_cadip/api/cadip_search.py b/services/cadip/rs_server_cadip/api/cadip_search.py index 15c002f43..51d644cd5 100644 --- a/services/cadip/rs_server_cadip/api/cadip_search.py +++ b/services/cadip/rs_server_cadip/api/cadip_search.py @@ -398,11 +398,13 @@ def process_session_search( # type: ignore # pylint: disable=too-many-argument ) except json.JSONDecodeError as exception: + logger.error(exception) raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=f"JSON Map Error: {exception}", ) from exception except ValueError as exception: + logger.error(exception) raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exception), diff --git a/services/cadip/rs_server_cadip/cadip_utils.py b/services/cadip/rs_server_cadip/cadip_utils.py index ce28ccbea..b73cf9265 100644 --- a/services/cadip/rs_server_cadip/cadip_utils.py +++ b/services/cadip/rs_server_cadip/cadip_utils.py @@ -30,12 +30,15 @@ import yaml from fastapi import HTTPException, status from rs_server_common.stac_api_common import map_stac_platform +from rs_server_common.utils.logging import Logging from stac_pydantic.shared import Asset DEFAULT_GEOM = {"geometry": "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))"} CADIP_CONFIG = Path(osp.realpath(osp.dirname(__file__))).parent / "config" search_yaml = CADIP_CONFIG / "cadip_search_config.yaml" +logger = Logging.default(__name__) + def read_conf(): """Used each time to read RSPY_CADIP_SEARCH_CONFIG config yaml.""" @@ -96,7 +99,7 @@ def from_session_expand_to_dag_serializer(input_sessions: List[eodag.EOProduct]) properties=update_product(product, session.properties["href"]), ) for session in input_sessions - for product in session.properties.get("Files", []) + for product in (session.properties.get("Files", []) or []) ] @@ -129,7 +132,8 @@ def validate_products(products: eodag.EOProduct): try: str(product) valid_eo_products.append(product) - except eodag.utils.exceptions.MisconfiguredError: + except eodag.utils.exceptions.MisconfiguredError as e: + logger.warn(e) continue return valid_eo_products diff --git a/services/common/rs_server_common/data_retrieval/eodag_provider.py b/services/common/rs_server_common/data_retrieval/eodag_provider.py index 89a8b1f00..6257377ac 100644 --- a/services/common/rs_server_common/data_retrieval/eodag_provider.py +++ b/services/common/rs_server_common/data_retrieval/eodag_provider.py @@ -24,14 +24,16 @@ import yaml from eodag import EODataAccessGateway, EOProduct, SearchResult from eodag.utils.exceptions import AuthenticationError, RequestError +from rs_server_common.utils.logging import Logging -from .provider import CreateProviderFailed, Provider, TimeRange - -# TODO: See TODO invalid token. Import 'from .provider SearchProductFailed' if needed +from .provider import CreateProviderFailed, Provider, SearchProductFailed, TimeRange # from fastapi import HTTPException +logger = Logging.default(__name__) + + class EodagProvider(Provider): """An EODAG provider. @@ -115,9 +117,9 @@ def _specific_search(self, between: TimeRange, **kwargs) -> Union[SearchResult, if session_id: # If request contains session id, map it to eodag parameter accordingly (SessionID for single, Ids for list) if isinstance(session_id, list): - mapped_search_args["SessionIds"] = ", ".join(session_id) + mapped_search_args["SessionIds"] = ", ".join(f"'{s}'" for s in session_id) elif isinstance(session_id, str): - mapped_search_args["SessionID"] = session_id + mapped_search_args["SessionID"] = "'" + session_id + "'" if sessions_search: # If request is for session search, handle platform - if any provided. @@ -128,9 +130,9 @@ def _specific_search(self, between: TimeRange, **kwargs) -> Union[SearchResult, mapped_search_args["SessionId"] = mapped_search_args.pop("SessionID") if platform: if isinstance(platform, list): - mapped_search_args["platforms"] = ", ".join(platform) + mapped_search_args["platforms"] = ", ".join(f"'{p}'" for p in platform) elif isinstance(platform, str): - mapped_search_args["platform"] = platform + mapped_search_args["platform"] = "'" + platform + "'" if between: # Since now both for files and sessions, time interval is optional, map it if provided. @@ -141,7 +143,11 @@ def _specific_search(self, between: TimeRange, **kwargs) -> Union[SearchResult, }, ) + # TODO: sort by publicationDate by default, but allow user to override it + mapped_search_args["sort_by"] = [("publicationDate", "DESC")] + try: + logger.info(f"Searching from {self.provider} with parameters {mapped_search_args}") # Start search -> user defined search params in mapped_search_args (id), pagination in kwargs (top, limit). products = self.client.search( **mapped_search_args, # type: ignore @@ -150,20 +156,22 @@ def _specific_search(self, between: TimeRange, **kwargs) -> Union[SearchResult, productType="S1_SAR_RAW" if "adgs" not in self.provider.lower() else "CAMS_GRF_AUX", **kwargs, ) - except RequestError: - # except RequestError as e: - # TODO invalid token: EODAG returns an exception with "FORBIDDEN" in e.args when the token key is invalid. - # Should we handle this specifically by raising an exception, or follow the current approach - # where any error results in returning an empty list? - # if e.args and "FORBIDDEN" in e.args[0]: - # raise SearchProductFailed( - # f"Can't search provider {self.provider} " "because the used token is not valid", - # ) from e - # Empty list if something goes wrong in eodag - return [] + except RequestError as e: + # invalid token: EODAG returns an exception with "FORBIDDEN" in e.args when the token key is invalid. + if e.args and "FORBIDDEN" in e.args[0]: + raise SearchProductFailed( + f"Can't search provider {self.provider} " "because the used token is not valid", + ) from e + raise SearchProductFailed(e) except AuthenticationError as exc: raise ValueError("EoDAG could not authenticate") from exc + if products.number_matched: + logger.info(f"Returned {products.number_matched} session from {self.provider}") + + if products.errors: + logger.error(f"Errors from {self.provider}: {products.errors}") + return products def download(self, product_id: str, to_file: Path) -> None: