From 51b73177e51dc9b239329caf59caa2e3192819cf Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 30 May 2024 10:44:41 -0800 Subject: [PATCH 1/6] OPERAS1Products now assigned to reuslts by checking meta collection-concept-id field --- CHANGELOG.md | 7 +++++++ asf_search/ASFProduct.py | 11 +++++++++++ asf_search/Products/ARIAS1GUNWProduct.py | 2 +- asf_search/Products/OPERAS1Product.py | 9 +++++++++ asf_search/search/baseline_search.py | 2 +- asf_search/search/search_generator.py | 9 ++++++--- 6 files changed, 35 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f315c6b5..93caf077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,13 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +------ +## [v7.1.2](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.0...v7.1.1) +### Fixed +- `OPERAS1Product` subclass now properly assigned to PGE v2.0.1 results +### Changed +- `ARIAS1GUNWProduct.is_ARIAS1GUNWProduct()` removed, replaced with `ASFProduct._is_subclass()` implementation + ------ ## [v7.1.1](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.0...v7.1.1) ### Changed diff --git a/asf_search/ASFProduct.py b/asf_search/ASFProduct.py index 0b252e87..e2681ff2 100644 --- a/asf_search/ASFProduct.py +++ b/asf_search/ASFProduct.py @@ -424,3 +424,14 @@ def umm_cast(f, v): return f(v) except TypeError: return None + + @staticmethod + def _is_subclass(item: Dict) -> bool: + """ + Used to determine which subclass to use for specific edge-cases when parsing results in search methods + (Currently implemented for ARIA and OPERA subclasses). + + params: + - item (dict): the CMR UMM-G item to read from + """ + raise NotImplementedError() diff --git a/asf_search/Products/ARIAS1GUNWProduct.py b/asf_search/Products/ARIAS1GUNWProduct.py index 34be38ce..ab477bfc 100644 --- a/asf_search/Products/ARIAS1GUNWProduct.py +++ b/asf_search/Products/ARIAS1GUNWProduct.py @@ -58,7 +58,7 @@ def get_default_baseline_product_type() -> None: return None @staticmethod - def is_ARIAS1GUNWProduct(item: Dict) -> bool: + def _is_subclass(item: Dict) -> bool: platform = ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName') if platform in ['SENTINEL-1A', 'SENTINEL-1B']: asf_platform = ASFProduct.umm_get(item['umm'], 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0) diff --git a/asf_search/Products/OPERAS1Product.py b/asf_search/Products/OPERAS1Product.py index 86af7bbe..67055875 100644 --- a/asf_search/Products/OPERAS1Product.py +++ b/asf_search/Products/OPERAS1Product.py @@ -19,6 +19,8 @@ class OPERAS1Product(S1Product): 'polarization': {'path': ['AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values']} # dual polarization is in list rather than a 'VV+VH' style format } + _subclass_concept_ids = { 'C1257995185-ASF', 'C1257995186-ASF', 'C1258354200-ASF', 'C1258354201-ASF', 'C1259974840-ASF', 'C1259976861-ASF', 'C1259981910-ASF', 'C1259982010-ASF', 'C2777436413-ASF', 'C2777443834-ASF', 'C2795135174-ASF', 'C2795135668-ASF','C1260721853-ASF', 'C1260721945-ASF', 'C2803501097-ASF', 'C2803501758-ASF' } + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): super().__init__(args, session) @@ -78,3 +80,10 @@ def get_sort_keys(self) -> Tuple[str, str]: return (self._read_property('validityStartDate', ''), keys[1]) return keys + + @staticmethod + def _is_subclass(item: Dict) -> bool: + # not all umm products have this field set, + # but when it's available it's convenient for fast matching + concept_id = item['meta'].get('collection-concept-id') + return concept_id in OPERAS1Product._subclass_concept_ids diff --git a/asf_search/search/baseline_search.py b/asf_search/search/baseline_search.py index e4299c31..f3b46a6c 100644 --- a/asf_search/search/baseline_search.py +++ b/asf_search/search/baseline_search.py @@ -52,7 +52,7 @@ def stack_from_product( stack.sort(key=lambda product: product.properties['temporalBaseline']) for warning in warnings: - ASF_LOGGER.warn(f'{warning}') + ASF_LOGGER.warning(f'{warning}') return stack diff --git a/asf_search/search/search_generator.py b/asf_search/search/search_generator.py index f2a9b772..10b64892 100644 --- a/asf_search/search/search_generator.py +++ b/asf_search/search/search_generator.py @@ -266,6 +266,9 @@ def as_ASFProduct(item: Dict, session: ASFSession) -> ASFProduct: :returns the granule as an object of type ASFProduct """ + if ASFProductType.OPERAS1Product._is_subclass(item): + return ASFProductType.OPERAS1Product(item, session=session) + product_type_key = _get_product_type_key(item) # if there's a direct entry in our dataset to product type dict @@ -283,7 +286,7 @@ def as_ASFProduct(item: Dict, session: ASFSession) -> ASFProduct: # If the platform exists, try to match it platform = _get_platform(item=item) - if ASFProductType.ARIAS1GUNWProduct.is_ARIAS1GUNWProduct(item=item): + if ASFProductType.ARIAS1GUNWProduct._is_subclass(item=item): return dataset_to_product_types.get('ARIA S1 GUNW')(item, session=session) elif (subclass := dataset_to_product_types.get(platform)) is not None: return subclass(item, session=session) @@ -306,10 +309,10 @@ def _get_product_type_key(item: Dict) -> str: collection_shortName = ASFProduct.umm_get(item['umm'], 'CollectionReference', 'ShortName') if collection_shortName is None: - platform = _get_platform(item=item) - if ASFProductType.ARIAS1GUNWProduct.is_ARIAS1GUNWProduct(item=item): + if ASFProductType.ARIAS1GUNWProduct._is_subclass(item=item): return 'ARIA S1 GUNW' + platform = _get_platform(item=item) return platform return collection_shortName From 24f7ed76dc2fcfab0452bd44c0df3ee3bdcb0875 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 31 May 2024 10:41:16 -0800 Subject: [PATCH 2/6] fixes typo in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93caf077..49103d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --> ------ -## [v7.1.2](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.0...v7.1.1) +## [v7.1.2](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.1...v7.1.2) ### Fixed - `OPERAS1Product` subclass now properly assigned to PGE v2.0.1 results ### Changed From baebae6cbbda26a9555f1958a418179b6daccb8c Mon Sep 17 00:00:00 2001 From: kim Date: Tue, 18 Jun 2024 11:17:07 -0800 Subject: [PATCH 3/6] bugfix: fixes missing polarization constants, RAW alias list includes SENTINEL-1B_RAW concept-id --- CHANGELOG.md | 6 ++++++ asf_search/CMR/datasets.py | 1 + asf_search/constants/POLARIZATION.py | 14 +++++++------- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49103d36..7c72c280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,12 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +------ +## [v7.1.3](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.2...v7.1.3) +### Fixed +- Adds missing values for polarization constants `DUAL_HH`, `DUAL_VV`, `DUAL_HV`, `DUAL_VH`, `HH_3SCAN`, `HH_4SCAN`, `HH_5SCAN` +- processingLevel `RAW` now includes `C1234413256-ASFDEV` in collection alias list (`SENTINEL-1B_RAW`'s collection for ASFDEV provider) + ------ ## [v7.1.2](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.1...v7.1.2) ### Fixed diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py index 7d49e089..6fc80eff 100644 --- a/asf_search/CMR/datasets.py +++ b/asf_search/CMR/datasets.py @@ -1168,6 +1168,7 @@ "C1234413238-ASFDEV", "C1327985647-ASF", "C1216244592-ASF", + "C1234413256-ASFDEV", ], "GRD_MD": [ "C1214471521-ASF", diff --git a/asf_search/constants/POLARIZATION.py b/asf_search/constants/POLARIZATION.py index 9966d92d..686ea32a 100644 --- a/asf_search/constants/POLARIZATION.py +++ b/asf_search/constants/POLARIZATION.py @@ -2,13 +2,13 @@ VV = 'VV' VV_VH = 'VV+VH' HH_HV = 'HH+HV' -DUAL_HH = '' -DUAL_VV = '' -DUAL_HV = '' -DUAL_VH = '' -HH_3SCAN = '' -HH_4SCAN = '' -HH_5SCAN = '' +DUAL_HH = 'DUAL HH' +DUAL_VV = 'DUAL VV' +DUAL_HV = 'DUAL HV' +DUAL_VH = 'DUAL VH' +HH_3SCAN = 'HH 3SCAN' +HH_4SCAN = 'HH 4SCAN' +HH_5SCAN = 'HH 5SCAN' QUAD = 'quadrature' HH_VV = 'HH+VV' HH_HV_VH_VV = 'HH+HV+VH+VV' From cfecc5406e273467d687b7799e7af6246adb1944 Mon Sep 17 00:00:00 2001 From: Cameron Showalter Date: Thu, 11 Jul 2024 12:07:22 -0800 Subject: [PATCH 4/6] Fix for python 3.12, escape characters throw warnings --- asf_search/CMR/translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index d564d9c7..7f6973f5 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -22,7 +22,7 @@ def translate_opts(opts: ASFSearchOptions) -> List: # intersectsWith, temporal, and other keys you don't want to escape, so keep whitelist instead for escape_commas in ["campaign"]: if escape_commas in dict_opts: - dict_opts[escape_commas] = dict_opts[escape_commas].replace(",", "\,") + dict_opts[escape_commas] = dict_opts[escape_commas].replace(",", "\\,") # Special case to unravel WKT field a little for compatibility if "intersectsWith" in dict_opts: From 84cf52a500569a343e0e455f0f0cdb83d4456136 Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 11 Jul 2024 14:39:33 -0800 Subject: [PATCH 5/6] using dateutil for date validation, ciso8601 optionalized --- CHANGELOG.md | 5 +++++ asf_search/CMR/translate.py | 8 ++++++-- asf_search/baseline/calc.py | 8 ++++++-- asf_search/baseline/stack.py | 8 +++++--- setup.py | 5 +++-- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c72c280..50b02c6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +------ +## [v7.1.4](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.3...v7.1.4) +### Changed +- replaces `ciso8601` package with `dateutil` for package wheel compatibility. `ciso8601` used when installed via `extra` dependency + ------ ## [v7.1.3](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.2...v7.1.3) ### Fixed diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index d564d9c7..2c5d2d93 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -9,9 +9,13 @@ from shapely.geometry.base import BaseGeometry from .field_map import field_map from .datasets import collections_per_platform -import ciso8601 import logging +try: + from ciso8601 import parse_datetime +except ImportError: + from dateutil.parser import parse as parse_datetime + def translate_opts(opts: ASFSearchOptions) -> List: # Need to add params which ASFSearchOptions cant support (like temporal), @@ -158,7 +162,7 @@ def try_parse_date(value: str) -> Optional[str]: return None try: - date = ciso8601.parse_datetime(value) + date = parse_datetime(value) except ValueError: return None diff --git a/asf_search/baseline/calc.py b/asf_search/baseline/calc.py index faa6442e..6e861d60 100644 --- a/asf_search/baseline/calc.py +++ b/asf_search/baseline/calc.py @@ -1,10 +1,14 @@ +from asf_search import ASFProduct from math import sqrt, cos, sin, radians from typing import List import numpy as np -from ciso8601 import parse_datetime -from asf_search import ASFProduct +try: + from ciso8601 import parse_datetime +except ImportError: + from dateutil.parser import parse as parse_datetime + # WGS84 constants a = 6378137 f = pow((1.0 - 1 / 298.257224), 2) diff --git a/asf_search/baseline/stack.py b/asf_search/baseline/stack.py index c443adae..4a686f7b 100644 --- a/asf_search/baseline/stack.py +++ b/asf_search/baseline/stack.py @@ -1,10 +1,12 @@ +from asf_search import ASFProduct, ASFStackableProduct, ASFSearchResults from typing import Tuple, List -from ciso8601 import parse_datetime import pytz - from .calc import calculate_perpendicular_baselines -from asf_search import ASFProduct, ASFStackableProduct, ASFSearchResults +try: + from ciso8601 import parse_datetime +except ImportError: + from dateutil.parser import parse as parse_datetime def get_baseline_from_stack(reference: ASFProduct, stack: ASFSearchResults) -> Tuple[ASFSearchResults, List[dict]]: warnings = [] diff --git a/setup.py b/setup.py index 36d31fc8..dff08b72 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ "importlib_metadata", "numpy", "dateparser", + "python-dateutil", "tenacity == 8.2.2", - "ciso8601" ] test_requirements = [ @@ -25,7 +25,8 @@ ] extra_requirements = [ - "remotezip>=0.10.0" + "remotezip>=0.10.0", + "ciso8601", ] From 30be1509fbc988f16cb6305bedbef986ea62c883 Mon Sep 17 00:00:00 2001 From: kim Date: Fri, 12 Jul 2024 08:28:07 -0800 Subject: [PATCH 6/6] update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c72c280..6adb6e89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +------ +## [v7.1.4](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.3...v7.1.4) +### Fixed +- Fixes syntax warning with escaped slash in `translate.py` + ------ ## [v7.1.3](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.2...v7.1.3) ### Fixed