diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index f1a747e5..855336a6 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -11,7 +11,7 @@ jobs: - name: Install Dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install .[test] + python3 -m pip install .[extras,test] - name: Run Tests run: python3 -m pytest -n auto --cov=asf_search --cov-report=xml --dont-run-file test_known_bugs . diff --git a/.gitignore b/.gitignore index 2dbe033b..0a85e8e1 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,5 @@ dmypy.json # VS Code .vscode/ +search_results.csv +search_results.metalink diff --git a/CHANGELOG.md b/CHANGELOG.md index 48892828..61e6a683 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,32 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - --> +------ +## [v.7.0.0](https://github.com/asfadmin/Discovery-asf_search/compare/v6.7.3...v.7.0.0) +### Added +- `ASFProduct` now has 13 sublcasses for different sub-products that correspond to datasets: + - `S1Product`, `S1BurstProduct`, `OPERAS1Product`, `ARIAS1GUNWProduct`, `ALOSProduct`, `RADARSATProduct`, `AIRSARProduct`, `ERSProduct`, `JERSProduct`, `UAVSARProduct`, `SIRCProduct`, `SEASATProduct`, `SMAPProduct` + - Each subclass defines relevant keys to pull from `umm` response, reducing the amount of irrelevant values in `properties` dict for certain product types +- Adds `collectionAlias` to `ASFSearchOptions` validator map as config param. Set to `False` to disable concept-id aliasing behaviour for `processingLevel` and `platform`. +- Adds warning when scenes in stack are missing state vectors, and logs baseline warnings with `ASF_LOGGER` +- Adds `OPERA-S1-CALIBRATION` entry to `dataset_collections` and corresponding `OPERA_S1_CALIBRATION` constant to `DATASET.py`, used to search for OPERA-S1 `CSLC` and `RTC` calibration data. + +### Changed +- `remotezip` is now an optional dependency of asf-search's pip and conda installs, (pip install example: `python3 -m pip install asf-search[extras]`). +- Constants are no longer top level import, are now accessible through respective modules +- `processingLevel` and `platform` are now aliased by collection concept-ids, (lists of concept ids by their processing levels/platforms viewable in `dataset.py`), improving search performance and dodging subquery system +- Baseline stacking no longer excludes products with missing state vectors from final stack, like SearchAPI +- `OPERA-S1` dataset no longer includes calibration data (moved to new dataset) +- Adds optional `ASFSession` constructor keyword arguments for new class variables: + - `edl_host` + - `edl_client_id` + - `asf_auth_host` + - `cmr_host` + - `cmr_collections` + - `auth_domains` +- `ASFSession` imports `asf_search.constants.INTERNAL` in constructor call +- `ASFSession` methods `auth_with_creds()`, `auth_with_token()`, and `rebuild_auth()` use new class variables instead of constants + ------ ## [v6.7.3](https://github.com/asfadmin/Discovery-asf_search/compare/v6.7.2...v6.7.3) ### Added diff --git a/asf_search/ASFProduct.py b/asf_search/ASFProduct.py index 3ceb4d02..f3a3df2b 100644 --- a/asf_search/ASFProduct.py +++ b/asf_search/ASFProduct.py @@ -1,4 +1,5 @@ import os +from typing import Any, Dict, Tuple, Type, List, final import warnings from shapely.geometry import shape, Point, Polygon, mapping import json @@ -6,31 +7,93 @@ from urllib import parse from asf_search import ASFSession, ASFSearchResults -from asf_search.ASFSearchOptions import ASFSearchOptions +from asf_search.ASFSearchOptions import ASFSearchOptions from asf_search.download import download_url -from asf_search.CMR import translate_product -from remotezip import RemoteZip - from asf_search.download.file_download_type import FileDownloadType -from asf_search import ASF_LOGGER +from asf_search.CMR.translate import try_parse_float, try_parse_int, try_round_float class ASFProduct: - def __init__(self, args: dict = {}, session: ASFSession = ASFSession()): + """ + The ASFProduct class is the base class for search results from asf-search. + Key props: + - properties: + - stores commonly acessed properties of the CMR UMM for convenience + - umm: + - The data portion of the CMR response + - meta: + - The metadata portion of the CMR response + - geometry: + - The geometry `{coordinates: [[lon, lat] ...], 'type': Polygon}` + - baseline: + - used for spatio-temporal baseline stacking, stores state vectors/ascending node time/insar baseline values when available (Not set in base ASFProduct class) + - See `S1Product` or `ALOSProduct` `get_baseline_calc_properties()` methods for implementation examples + + Key methods: + - `download()` + - `stack()` + - `remotezip()` + + + """ + @classmethod + def get_classname(cls): + return cls.__name__ + + _base_properties = { + # min viable product + 'centerLat': {'path': ['AdditionalAttributes', ('Name', 'CENTER_LAT'), 'Values', 0], 'cast': try_parse_float}, + 'centerLon': {'path': ['AdditionalAttributes', ('Name', 'CENTER_LON'), 'Values', 0], 'cast': try_parse_float}, + 'stopTime': {'path': ['TemporalExtent', 'RangeDateTime', 'EndingDateTime']}, # primary search results sort key + 'fileID': {'path': ['GranuleUR']}, # secondary search results sort key + 'flightDirection': {'path': [ 'AdditionalAttributes', ('Name', 'ASCENDING_DESCENDING'), 'Values', 0]}, + 'pathNumber': {'path': ['AdditionalAttributes', ('Name', 'PATH_NUMBER'), 'Values', 0], 'cast': try_parse_int}, + 'processingLevel': {'path': [ 'AdditionalAttributes', ('Name', 'PROCESSING_TYPE'), 'Values', 0]}, + + # commonly used + 'url': {'path': [ 'RelatedUrls', ('Type', 'GET DATA'), 'URL']}, + 'startTime': {'path': [ 'TemporalExtent', 'RangeDateTime', 'BeginningDateTime']}, + 'sceneName': {'path': [ 'DataGranule', 'Identifiers', ('IdentifierType', 'ProducerGranuleId'), 'Identifier']}, + 'browse': {'path': ['RelatedUrls', ('Type', [('GET RELATED VISUALIZATION', 'URL')])]}, + 'platform': {'path': [ 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0]}, + 'bytes': {'path': [ 'AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, + 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0], 'cast': try_parse_int}, # overloaded by S1, ALOS, and ERS + 'granuleType': {'path': [ 'AdditionalAttributes', ('Name', 'GRANULE_TYPE'), 'Values', 0]}, + 'orbit': {'path': [ 'OrbitCalculatedSpatialDomains', 0, 'OrbitNumber'], 'cast': try_parse_int}, + 'polarization': {'path': [ 'AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values', 0]}, + 'processingDate': {'path': [ 'DataGranule', 'ProductionDateTime'], }, + 'sensor': {'path': [ 'Platforms', 0, 'Instruments', 0, 'ShortName'], }, + } + """ + _base_properties dictionary, mapping readable property names to paths and optional type casting + + entries are organized as such: + - `PROPERTY_NAME`: The name the property should be called in `ASFProduct.properties` + - `path`: the expected path in the CMR UMM json granule response as a list + - `cast`: (optional): the optional type casting method + + Defining `_base_properties` in subclasses allows for defining custom properties or overiding existing ones. + See `S1Product.get_property_paths()` on how subclasses are expected to + combine `ASFProduct._base_properties` with their own separately defined `_base_properties` + """ + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): self.meta = args.get('meta') self.umm = args.get('umm') - translated = translate_product(args) + translated = self.translate_product(args) self.properties = translated['properties'] self.geometry = translated['geometry'] - self.baseline = translated['baseline'] + self.baseline = None self.session = session def __str__(self): return json.dumps(self.geojson(), indent=2, sort_keys=True) - def geojson(self) -> dict: + def geojson(self) -> Dict: + """Returns ASFProduct object as a geojson formatted dictionary, with `type`, `geometry`, and `properties` keys""" return { 'type': 'Feature', 'geometry': self.geometry, @@ -52,54 +115,54 @@ def download(self, path: str, filename: str = None, session: ASFSession = None, if filename is not None: multiple_files = ( - (fileType == FileDownloadType.ADDITIONAL_FILES and len(self.properties['additionalUrls']) > 1) + (fileType == FileDownloadType.ADDITIONAL_FILES and len(self.properties['additionalUrls']) > 1) or fileType == FileDownloadType.ALL_FILES ) if multiple_files: warnings.warn(f"Attempting to download multiple files for product, ignoring user provided filename argument \"{filename}\", using default.") else: default_filename = filename - + if session is None: session = self.session urls = [] - def get_additional_urls(): - output = [] - for url in self.properties['additionalUrls']: - if self.properties['processingLevel'] == 'BURST': - # Burst XML filenames are just numbers, this makes it more indentifiable - file_name = '.'.join(default_filename.split('.')[:-1]) + url.split('.')[-1] - else: - # otherwise just use the name found in the url - file_name = os.path.split(parse.urlparse(url).path)[1] - urls.append((f"{file_name}", url)) - - return output - if fileType == FileDownloadType.DEFAULT_FILE: urls.append((default_filename, self.properties['url'])) elif fileType == FileDownloadType.ADDITIONAL_FILES: - urls.extend(get_additional_urls()) + urls.extend(self._get_additional_filenames_and_urls(default_filename)) elif fileType == FileDownloadType.ALL_FILES: urls.append((default_filename, self.properties['url'])) - urls.extend(get_additional_urls()) + urls.extend(self._get_additional_filenames_and_urls(default_filename)) else: raise ValueError("Invalid FileDownloadType provided, the valid types are 'DEFAULT_FILE', 'ADDITIONAL_FILES', and 'ALL_FILES'") for filename, url in urls: download_url(url=url, path=path, filename=filename, session=session) + def _get_additional_filenames_and_urls( + self, + default_filename: str = None # for subclasses without fileName in url (see S1BurstProduct implementation) + ) -> List[Tuple[str, str]]: + return [(self._parse_filename_from_url(url), url) for url in self.properties.get('additionalUrls', [])] + + def _parse_filename_from_url(self, url: str) -> str: + file_path = os.path.split(parse.urlparse(url).path) + filename = file_path[1] + return filename + def stack( self, - opts: ASFSearchOptions = None + opts: ASFSearchOptions = None, + useSubclass: Type['ASFProduct'] = None ) -> ASFSearchResults: """ Builds a baseline stack from this product. :param opts: An ASFSearchOptions object describing the search parameters to be used. Search parameters specified outside this object will override in event of a conflict. - + :param ASFProductSubclass: An ASFProduct subclass constructor. + :return: ASFSearchResults containing the stack, with the addition of baseline values (temporal, perpendicular) attached to each ASFProduct. """ from .search.baseline_search import stack_from_product @@ -107,17 +170,15 @@ def stack( if opts is None: opts = ASFSearchOptions(session=self.session) - return stack_from_product(self, opts=opts) + return stack_from_product(self, opts=opts, ASFProductSubclass=useSubclass) - def get_stack_opts(self) -> ASFSearchOptions: + def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: """ Build search options that can be used to find an insar stack for this product :return: ASFSearchOptions describing appropriate options for building a stack from this product """ - from .search.baseline_search import get_stack_opts - - return get_stack_opts(reference=self) + return None def centroid(self) -> Point: """ @@ -132,12 +193,189 @@ def centroid(self) -> Point: return Polygon(unwrapped_coords).centroid - def remotezip(self, session: ASFSession) -> RemoteZip: + def remotezip(self, session: ASFSession) -> 'RemoteZip': """Returns a RemoteZip object which can be used to download a part of an ASFProduct's zip archive. (See example in examples/5-Download.ipynb) + requires installing optional dependencies via pip or conda to use the `remotezip` package: + + `python3 -m pip install asf-search[extras]` + :param session: an authenticated ASFSession """ from .download.download import remotezip return remotezip(self.properties['url'], session=session) + + def _read_umm_property(self, umm: Dict, mapping: Dict) -> Any: + value = self.umm_get(umm, *mapping['path']) + if mapping.get('cast') is None: + return value + + return self.umm_cast(mapping['cast'], value) + + def translate_product(self, item: Dict) -> Dict: + """ + Generates `properties` and `geometry` from the CMR UMM response + """ + try: + coordinates = item['umm']['SpatialExtent']['HorizontalSpatialDomain']['Geometry']['GPolygons'][0]['Boundary']['Points'] + coordinates = [[c['Longitude'], c['Latitude']] for c in coordinates] + geometry = {'coordinates': [coordinates], 'type': 'Polygon'} + except KeyError: + geometry = {'coordinates': None, 'type': 'Polygon'} + + umm = item.get('umm') + + properties = { + prop: self._read_umm_property(umm, umm_mapping) + for prop, umm_mapping in self.get_property_paths().items() + } + + if properties.get('url') is not None: + properties['fileName'] = properties['url'].split('/')[-1] + else: + properties['fileName'] = None + + # Fallbacks + if properties.get('beamModeType') is None: + properties['beamModeType'] = self.umm_get(umm, 'AdditionalAttributes', ('Name', 'BEAM_MODE'), 'Values', 0) + + if properties.get('platform') is None: + properties['platform'] = self.umm_get(umm, 'Platforms', 0, 'ShortName') + + return {'geometry': geometry, 'properties': properties, 'type': 'Feature'} + + # ASFProduct subclasses define extra/override param key + UMM pathing here + @staticmethod + def get_property_paths() -> Dict: + """ + Returns _base_properties of class, subclasses such as `S1Product` (or user provided subclasses) can override this to + define which properties they want in their subclass's properties dict. + + (See `S1Product.get_property_paths()` for example of combining _base_properties of multiple classes) + + :returns dictionary, {`PROPERTY_NAME`: {'path': [umm, path, to, value], 'cast (optional)': Callable_to_cast_value}, ...} + """ + return ASFProduct._base_properties + + def get_sort_keys(self) -> Tuple: + """ + Returns tuple of primary and secondary date values used for sorting final search results + """ + return (self.properties.get('stopTime'), self.properties.get('fileID', 'sceneName')) + + @final + @staticmethod + def umm_get(item: Dict, *args): + """ + Used to search for values in CMR UMM + + :param item: the umm dict returned from CMR + :param *args: the expected path to the value + + Example case: + "I want to grab the polarization from the granule umm" + ``` + item = { + 'AdditionalAttributes': [ + { + 'Name': 'POLARIZATION', + 'Values': ['VV', 'VH'] + }, + ... + ], + ... + } + ``` + + The path provided to *args would look like this: + ``` + 'AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values', 0 + result: 'VV' + ``` + + - `'AdditionalAttributes'` acts like item['AdditionalAttributes'], which is a list of dictionaries + + - Since `AdditionalAttributes` is a LIST of dictionaries, we search for a dict with the key value pair, + `('Name', 'POLARIZATION')` + + - If found, we try to access that dictionary's `Values` key + - Since `Values` is a list, we can access the first index `0` (in this case, 'VV') + + --- + + If you want more of the umm, simply reduce how deep you search: + Example: "I need BOTH polarizations (`OPERAS1Product` does this, noticed the omitted `0`) + + ``` + 'AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values' + result: ['VV', 'VH'] + ``` + + --- + + Example: "I need the ENTIRE POLARIZATION dict" + + ``` + 'AdditionalAttributes', ('Name', 'POLARIZATION') + result: { + 'Name': 'POLARIZATION', + 'Values': ['VV', 'VH'] + } + ``` + + --- + + ADVANCED: + Sometimes there are multiple dictionaries in a list that have the same key value pair we're searching for + (See `OPERAS1Product` umm under `RelatedUrls`). This means we can miss values since we're only grabbing the first match + depending on how the umm is organized. There is a way to get ALL data that matches our key value criteria. + + Example: "I need ALL `URL` values for dictionaries in `RelatedUrls` where `Type` is `GET DATA`" (See in use in `OPERAS1Product` class) + ``` + 'RelatedUrls', ('Type', [('GET DATA', 'URL')]), 0 + ``` + """ + if item is None: + return None + for key in args: + if isinstance(key, int): + item = item[key] if key < len(item) else None + elif isinstance(key, tuple): + (a, b) = key + if isinstance(b, List): + output = [] + b = b[0] + for child in item: + if ASFProduct.umm_get(child, key[0]) == b[0]: + output.append(ASFProduct.umm_get(child, b[1])) + if len(output): + return output + + return None + + found = False + for child in item: + if ASFProduct.umm_get(child, a) == b: + item = child + found = True + break + if not found: + return None + else: + item = item.get(key) + if item is None: + return None + if item in [None, 'NA', 'N/A', '']: + item = None + return item + + @final + @staticmethod + def umm_cast(f, v): + """Tries to cast value v by callable f, returns None if it fails""" + try: + return f(v) + except TypeError: + return None diff --git a/asf_search/ASFSearchOptions/config.py b/asf_search/ASFSearchOptions/config.py index 13ebb430..6b02e947 100644 --- a/asf_search/ASFSearchOptions/config.py +++ b/asf_search/ASFSearchOptions/config.py @@ -5,4 +5,5 @@ 'host': INTERNAL.CMR_HOST, 'provider': INTERNAL.DEFAULT_PROVIDER, 'session': ASFSession(), + 'collectionAlias': True } diff --git a/asf_search/ASFSearchOptions/validator_map.py b/asf_search/ASFSearchOptions/validator_map.py index 38803de3..72fdd0a4 100644 --- a/asf_search/ASFSearchOptions/validator_map.py +++ b/asf_search/ASFSearchOptions/validator_map.py @@ -66,5 +66,6 @@ def validate(key, value): # Config parameters Parser 'session': parse_session, 'host': parse_string, - 'provider': parse_string + 'provider': parse_string, + 'collectionAlias': bool, } diff --git a/asf_search/ASFSearchResults.py b/asf_search/ASFSearchResults.py index 1030e948..77ef7f94 100644 --- a/asf_search/ASFSearchResults.py +++ b/asf_search/ASFSearchResults.py @@ -79,6 +79,24 @@ def raise_if_incomplete(self) -> None: ASF_LOGGER.error(msg) raise ASFSearchError(msg) + def get_products_by_subclass_type(self) -> dict: + """ + Organizes results into dictionary by ASFProduct subclass name + : return: Dict of ASFSearchResults, organized by ASFProduct subclass names + """ + subclasses = {} + + for product in self.data: + product_type = product.get_classname() + + if subclasses.get(product_type) is None: + subclasses[product_type] = ASFSearchResults([]) + + subclasses[product_type].append(product) + + return subclasses + def _download_product(args) -> None: product, path, session, fileType = args product.download(path=path, session=session, fileType=fileType) + diff --git a/asf_search/ASFSession.py b/asf_search/ASFSession.py index 951a46a2..dddf4494 100644 --- a/asf_search/ASFSession.py +++ b/asf_search/ASFSession.py @@ -3,12 +3,36 @@ from requests.utils import get_netrc_auth import http.cookiejar from asf_search import __name__ as asf_name, __version__ as asf_version -from asf_search.constants import EDL_CLIENT_ID, EDL_HOST, ASF_AUTH_HOST, AUTH_DOMAINS, CMR_HOST, CMR_COLLECTIONS from asf_search.exceptions import ASFAuthenticationError class ASFSession(requests.Session): - - def __init__(self): + def __init__(self, + edl_host: str = None, + edl_client_id: str = None, + asf_auth_host: str = None, + cmr_host: str = None, + cmr_collections: str = None, + auth_domains: str = None + ): + """ + ASFSession is a subclass of `requests.Session`, and is meant to ease downloading ASF hosted data by simplifying logging in to Earthdata Login. + To create an EDL account, see here: https://urs.earthdata.nasa.gov/users/new + + ASFSession provides three built-in methods for authorizing downloads: + - EDL Username and Password: `auth_with_creds()` + - EDL Token: `auth_with_token()` + - Authenticated cookiejars: `auth_with_cookiejar()` + + `edl_host`: the Earthdata login endpoint used by auth_with_creds(). Defaults to `asf_search.constants.INTERNAL.EDL_HOST` + `edl_client_id`: The Earthdata Login client ID for this package. Defaults to `asf_search.constants.INTERNAL.EDL_CLIENT_ID` + `asf_auth_host`: the ASF auth endpoint . Defaults to `asf_search.constants.INTERNAL.ASF_AUTH_HOST` + `cmr_host`: the base CMR endpoint to test EDL login tokens against. Defaults to `asf_search.constants.INTERNAL.CMR_HOST` + `cmr_collections`: the CMR endpoint path login tokens will be tested against. Defaults to `asf_search.constants.INTERNAL.CMR_COLLECTIONS` + `auth_domains`: the list of authorized endpoints that are allowed to pass auth credentials. Defaults to `asf_search.constants.INTERNAL.AUTH_DOMAINS`. Authorization headers WILL NOT be stripped from the session object when redirected through these domains. + + More information on Earthdata Login can be found here: + https://urs.earthdata.nasa.gov/documentation/faq + """ super().__init__() user_agent = '; '.join([ f'Python/{platform.python_version()}', @@ -18,6 +42,15 @@ def __init__(self): self.headers.update({'User-Agent': user_agent}) # For all hosts self.headers.update({'Client-Id': f"{asf_name}_v{asf_version}"}) # For CMR + from asf_search.constants import INTERNAL + + self.edl_host = INTERNAL.EDL_HOST if edl_host is None else edl_host + self.edl_client_id = INTERNAL.EDL_CLIENT_ID if edl_client_id is None else edl_client_id + self.asf_auth_host = INTERNAL.ASF_AUTH_HOST if asf_auth_host is None else asf_auth_host + self.cmr_host = INTERNAL.CMR_HOST if cmr_host is None else cmr_host + self.cmr_collections = INTERNAL.CMR_COLLECTIONS if cmr_collections is None else cmr_collections + self.auth_domains = INTERNAL.AUTH_DOMAINS if auth_domains is None else auth_domains + def __eq__(self, other): return self.auth == other.auth \ and self.headers == other.headers \ @@ -33,7 +66,7 @@ def auth_with_creds(self, username: str, password: str): :return ASFSession: returns self for convenience """ - login_url = f'https://{EDL_HOST}/oauth/authorize?client_id={EDL_CLIENT_ID}&response_type=code&redirect_uri=https://{ASF_AUTH_HOST}/login' + login_url = f'https://{self.edl_host}/oauth/authorize?client_id={self.edl_client_id}&response_type=code&redirect_uri=https://{self.asf_auth_host}/login' self.auth = (username, password) self.get(login_url) @@ -53,7 +86,7 @@ def auth_with_token(self, token: str): """ self.headers.update({'Authorization': 'Bearer {0}'.format(token)}) - url = f"https://{CMR_HOST}{CMR_COLLECTIONS}" + url = f"https://{self.cmr_host}{self.cmr_collections}" response = self.get(url) if not 200 <= response.status_code <= 299: @@ -95,8 +128,8 @@ def rebuild_auth(self, prepared_request: requests.Request, response: requests.Re redirect_domain = '.'.join(self._get_domain(url).split('.')[-3:]) if (original_domain != redirect_domain - and (original_domain not in AUTH_DOMAINS - or redirect_domain not in AUTH_DOMAINS)): + and (original_domain not in self.auth_domains + or redirect_domain not in self.auth_domains)): del headers['Authorization'] new_auth = get_netrc_auth(url) if self.trust_env else None @@ -105,3 +138,18 @@ def rebuild_auth(self, prepared_request: requests.Request, response: requests.Re def _get_domain(self, url: str): return requests.utils.urlparse(url).hostname + + # multi-processing does an implicit copy of ASFSession objects, + # this ensures ASFSession class variables are included + def __getstate__(self): + state = super().__getstate__() + state = { + **state, + 'edl_host': self.edl_host, + 'edl_client_id': self.edl_client_id, + 'asf_auth_host': self.asf_auth_host, + 'cmr_host': self.cmr_host, + 'cmr_collections': self.cmr_collections, + 'auth_domains': self.auth_domains + } + return state diff --git a/asf_search/ASFStackableProduct.py b/asf_search/ASFStackableProduct.py new file mode 100644 index 00000000..60c3830e --- /dev/null +++ b/asf_search/ASFStackableProduct.py @@ -0,0 +1,75 @@ +from enum import Enum +import copy +from typing import Dict, Union +from asf_search import ASFSession, ASFProduct +from asf_search.ASFSearchOptions import ASFSearchOptions +from asf_search.exceptions import ASFBaselineError + + +class ASFStackableProduct(ASFProduct): + """ + Used for ERS-1 and ERS-2 products + + ASF ERS-1 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-1/ + ASF ERS-2 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-2/ + """ + _base_properties = { + } + + class BaselineCalcType(Enum): + """ + Defines how asf-search will calculate perpendicular baseline for products of this subclass + """ + PRE_CALCULATED = 0 + """Has pre-calculated insarBaseline value that will be used for perpendicular calculations""" + CALCULATED = 1 + """Uses position/velocity state vectors and ascending node time for perpendicular calculations""" + + + baseline_type = BaselineCalcType.PRE_CALCULATED + """Determines how asf-search will attempt to stack products of this type.""" + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + self.baseline = self.get_baseline_calc_properties() + + def get_baseline_calc_properties(self) -> Dict: + insarBaseline = self.umm_cast(float, self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'INSAR_BASELINE'), 'Values', 0)) + + if insarBaseline is None: + return None + + return { + 'insarBaseline': insarBaseline + } + + def get_stack_opts(self, opts: ASFSearchOptions = None): + stack_opts = (ASFSearchOptions() if opts is None else copy(opts)) + stack_opts.processingLevel = self.get_default_baseline_product_type() + + if self.properties.get('insarStackId') in [None, 'NA', 0, '0']: + raise ASFBaselineError(f'Requested reference product needs a baseline stack ID but does not have one: {self.properties["fileID"]}') + + stack_opts.insarStackId = self.properties['insarStackId'] + return stack_opts + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFProduct.get_property_paths(), + **ASFStackableProduct._base_properties + } + + def is_valid_reference(self): + # we don't stack at all if any of stack is missing insarBaseline, unlike stacking S1 products(?) + if 'insarBaseline' not in self.baseline: + raise ValueError('No baseline values available for precalculated dataset') + + return True + + @staticmethod + def get_default_baseline_product_type() -> Union[str, None]: + """ + Returns the product type to search for when building a baseline stack. + """ + return None diff --git a/asf_search/CMR/__init__.py b/asf_search/CMR/__init__.py index 905e8eb3..50690b77 100644 --- a/asf_search/CMR/__init__.py +++ b/asf_search/CMR/__init__.py @@ -1,5 +1,5 @@ from .MissionList import get_campaigns from .subquery import build_subqueries -from .translate import translate_product, translate_opts, get_additional_fields +from .translate import translate_opts from .field_map import field_map -from .datasets import dataset_collections \ No newline at end of file +from .datasets import dataset_collections, collections_per_platform, collections_by_processing_level, get_concept_id_alias, get_dataset_concept_ids diff --git a/asf_search/CMR/datasets.py b/asf_search/CMR/datasets.py index 40abf2fe..d8415850 100644 --- a/asf_search/CMR/datasets.py +++ b/asf_search/CMR/datasets.py @@ -1,3 +1,6 @@ +from typing import List + + dataset_collections = { "SENTINEL-1": { "SENTINEL-1A_SLC": ["C1214470488-ASF", "C1205428742-ASF", "C1234413245-ASFDEV"], @@ -162,10 +165,10 @@ "OPERA_L2_RTC-S1-STATIC_PROVISIONAL_V0": ["C1258354201-ASF"], "OPERA_L2_RTC-S1-STATIC_V1": ["C1259981910-ASF", "C2795135174-ASF"], "OPERA_L2_RTC-S1_PROVISIONAL_V0": ["C1257995186-ASF"], - "OPERA_L2_CSLC-S1-STATIC_CALVAL_V1": ["C1260726384-ASF", "C2803502140-ASF"], + }, + "OPERA-S1-CALIBRATION": { "OPERA_L2_CSLC-S1_CALVAL_V1": ["C1260721945-ASF", "C2803501758-ASF"], "OPERA_L2_RTC-S1_CALVAL_V1": ["C1260721853-ASF", "C2803501097-ASF"], - "OPERA_L2_RTC-S1-STATIC_CALVAL_V1": ["C1260726378-ASF", "C2803500298-ASF"], }, "SLC-BURST": {"SENTINEL-1_BURSTS": ["C2709161906-ASF", "C1257024016-ASF"]}, "ALOS PALSAR": { @@ -227,7 +230,7 @@ "SENTINEL-1_INTERFEROGRAMS_UNWRAPPED_PHASE": [ "C1595765183-ASF", "C1225776659-ASF", - ], + ] }, "SMAP": { "SPL1A_RO_METADATA_003": ["C1243122884-ASF", "C1233103964-ASF"], @@ -327,7 +330,7 @@ } collections_per_platform = { - "Sentinel-1A": [ + "SENTINEL-1A": [ "C1214470488-ASF", "C1214470533-ASF", "C1214470576-ASF", @@ -413,7 +416,7 @@ "C1244598379-ASFDEV", "C1240784657-ASFDEV", ], - "Sentinel-1B": [ + "SENTINEL-1B": [ "C1327985661-ASF", "C1327985645-ASF", "C1595422627-ASF", @@ -729,7 +732,7 @@ } -collections_by_processing_level: { +collections_by_processing_level = { "SLC": [ "C1214470488-ASF", "C1205428742-ASF", @@ -1071,3 +1074,40 @@ "SLOPE": ["C1214408428-ASF", "C1210599503-ASF"], "STOKES": ["C1214419355-ASF", "C1210599673-ASF"], } + +#################### Helper Methods #################### + +def get_concept_id_alias(param_list: List[str], collections_dict: dict) -> List[str]: + """ + param: param_list (List[str]): list of search values to alias + param: collections_dict (dict): The search value to concept-id dictionary to read from + + returns List[str]: Returns a list of concept-ids that correspond to the given list of search values + If any of the search values are not keys in the collections_dict, this will instead returns an empty list. + """ + concept_id_aliases = [] + for param in param_list: + if alias := collections_dict.get(param): + concept_id_aliases.extend(alias) + else: + return [] + + return concept_id_aliases + +def get_dataset_concept_ids(datasets: List[str]) -> List[str]: + """ + Returns concept-ids for provided dataset(s) + If an invalid datset is provided a ValueError is raised + + :param `datasets` (`List[str]`): a list of datasets to grab concept-ids for + :returns `List[str]`: the list of concept-ids associated with the given datasets + """ + output = [] + for dataset in datasets: + if collections_by_short_name := dataset_collections.get(dataset): + for concept_ids in collections_by_short_name.values(): + output.extend(concept_ids) + else: + raise ValueError(f'Could not find dataset named "{dataset}" provided for dataset keyword.') + + return output \ No newline at end of file diff --git a/asf_search/CMR/subquery.py b/asf_search/CMR/subquery.py index 42bc49cd..e4a1b3c6 100644 --- a/asf_search/CMR/subquery.py +++ b/asf_search/CMR/subquery.py @@ -1,57 +1,111 @@ -from typing import List +from typing import List, Optional, Tuple import itertools from copy import copy from asf_search.ASFSearchOptions import ASFSearchOptions from asf_search.constants import CMR_PAGE_SIZE +from asf_search.CMR.datasets import collections_by_processing_level, collections_per_platform, dataset_collections, get_concept_id_alias, get_dataset_concept_ids +from numpy import intersect1d, union1d def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]: """ Build a list of sub-queries using the cartesian product of all the list parameters described by opts :param opts: The search options to split into sub-queries - :return list: A list of ASFSearchOptions objects """ params = dict(opts) # Break out two big list offenders into manageable chunks - if params.get('granule_list') is not None: - params['granule_list'] = chunk_list(params['granule_list'], CMR_PAGE_SIZE) - if params.get('product_list') is not None: - params['product_list'] = chunk_list(params['product_list'], CMR_PAGE_SIZE) + for chunked_key in ['granule_list', 'product_list']: + if params.get(chunked_key) is not None: + params[chunked_key] = chunk_list(params[chunked_key], CMR_PAGE_SIZE) list_param_names = ['platform', 'season', 'collections', 'dataset'] # these parameters will dodge the subquery system skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL - params = dict([ (k, v) for k, v in params.items() if k not in skip_param_names ]) - + collections, aliased_keywords = get_keyword_concept_ids(params, opts.collectionAlias) + params['collections'] = list(union1d(collections, params.get('collections', []))) + + for keyword in [*skip_param_names, *aliased_keywords]: + params.pop(keyword, None) + subquery_params, list_params = {}, {} - for k, v in params.items(): - if k in list_param_names: - list_params[k] = v + for key, value in params.items(): + if key in list_param_names: + list_params[key] = value else: - subquery_params[k] = v - + subquery_params[key] = value + sub_queries = cartesian_product(subquery_params) + return [_build_subquery(query, opts, list_params) for query in sub_queries] - final_sub_query_opts = [] - for query in sub_queries: - q = dict() - for p in query: - q.update(p) - q['provider'] = opts.provider - q['host'] = opts.host - q['session'] = copy(opts.session) - for key in list_params.keys(): - q[key] = list_params[key] - - final_sub_query_opts.append(ASFSearchOptions(**q)) - - return final_sub_query_opts +def _build_subquery(query: List[Tuple[dict]], opts: ASFSearchOptions, list_params: dict) -> ASFSearchOptions: + """ + Composes query dict and list params into new ASFSearchOptions object + param: query: the cartesian search query options + param: opts: the search options to pull config options from (provider, host, session) + param: list_params: the subquery parameters + """ + q = dict() + for p in query: + q.update(p) + return ASFSearchOptions( + **q, + provider= opts.provider, + host= opts.host, + session= copy(opts.session), + **list_params + ) + +def get_keyword_concept_ids(params: dict, use_collection_alias: bool=True) -> dict: + """ + Gets concept-ids for dataset, platform, processingLevel keywords + processingLevel is scoped by dataset or platform concept-ids when available + + : param params: search parameter dictionary pre-CMR translation + : param use_collection_alias: whether or not to alias platform and processingLevel with concept-ids + : returns two lists: + - list of concept-ids for dataset, platform, and processingLevel + - list of aliased keywords to remove from final parameters + """ + collections = [] + aliased_keywords = [] + + if use_collection_alias: + if 'processingLevel' in params.keys(): + collections = get_concept_id_alias(params.get('processingLevel'), collections_by_processing_level) + if len(collections): + aliased_keywords.append('processingLevel') + + if 'platform' in params.keys(): + platform_concept_ids = get_concept_id_alias( + [platform.upper() for platform in params.get('platform')], + collections_per_platform + ) + if len(platform_concept_ids): + aliased_keywords.append('platform') + collections = _get_intersection(platform_concept_ids, collections) + + if 'dataset' in params.keys(): + aliased_keywords.append('dataset') + dataset_concept_ids = get_dataset_concept_ids(params.get('dataset')) + collections = _get_intersection(dataset_concept_ids, collections) + + return collections, aliased_keywords +def _get_intersection(keyword_concept_ids: List[str], intersecting_ids: List[str]) -> List[str]: + """ + Returns the intersection between two lists. If the second list is empty the first list + is return unchaged + """ + if len(intersecting_ids): + return list(intersect1d(intersecting_ids, keyword_concept_ids)) + + return keyword_concept_ids + def chunk_list(source: List, n: int) -> List: """ Breaks a longer list into a list of lists, each of length n @@ -70,7 +124,7 @@ def cartesian_product(params): return p -def format_query_params(params): +def format_query_params(params) -> List[List[dict]]: listed_params = [] for param_name, param_val in params.items(): @@ -80,7 +134,7 @@ def format_query_params(params): return listed_params -def translate_param(param_name, param_val): +def translate_param(param_name, param_val) -> List[dict]: param_list = [] if not isinstance(param_val, list): diff --git a/asf_search/CMR/translate.py b/asf_search/CMR/translate.py index cba982d9..f6c79049 100644 --- a/asf_search/CMR/translate.py +++ b/asf_search/CMR/translate.py @@ -1,18 +1,19 @@ from datetime import datetime -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from asf_search.ASFSearchOptions import ASFSearchOptions +from asf_search.CMR.datasets import get_concept_id_alias from asf_search.constants import CMR_PAGE_SIZE import re from shapely import wkt from shapely.geometry import Polygon from shapely.geometry.base import BaseGeometry from .field_map import field_map -from .datasets import dataset_collections +from .datasets import dataset_collections, collections_per_platform import logging -def translate_opts(opts: ASFSearchOptions) -> list: +def translate_opts(opts: ASFSearchOptions) -> List: # Need to add params which ASFSearchOptions cant support (like temporal), # so use a dict to avoid the validate_params logic: dict_opts = dict(opts) @@ -48,19 +49,6 @@ def translate_opts(opts: ASFSearchOptions) -> list: if any(key in dict_opts for key in ['start', 'end', 'season']): dict_opts = fix_date(dict_opts) - if 'dataset' in dict_opts: - if 'collections' not in dict_opts: - dict_opts['collections'] = [] - - for dataset in dict_opts['dataset']: - if collections_by_short_name := dataset_collections.get(dataset): - for concept_ids in collections_by_short_name.values(): - dict_opts['collections'].extend(concept_ids) - else: - raise ValueError(f'Could not find dataset named "{dataset}" provided for dataset keyword.') - - dict_opts.pop('dataset') - # convert the above parameters to a list of key/value tuples cmr_opts = [] for (key, val) in dict_opts.items(): @@ -99,15 +87,10 @@ def translate_opts(opts: ASFSearchOptions) -> list: return cmr_opts - def should_use_asf_frame(cmr_opts): asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS'] - asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2'] - asf_frame_collections = [] - for dataset in asf_frame_datasets: - for concept_ids in dataset_collections.get(dataset).values(): - asf_frame_collections.extend(concept_ids) + asf_frame_collections = get_concept_id_alias(asf_frame_platforms, collections_per_platform) return any([ p[0] == 'platform[]' and p[1].upper() in asf_frame_platforms @@ -144,197 +127,25 @@ def use_asf_frame(cmr_opts): return cmr_opts - -def translate_product(item: dict) -> dict: - try: - coordinates = item['umm']['SpatialExtent']['HorizontalSpatialDomain']['Geometry']['GPolygons'][0]['Boundary']['Points'] - coordinates = [[c['Longitude'], c['Latitude']] for c in coordinates] - geometry = {'coordinates': [coordinates], 'type': 'Polygon'} - except KeyError as e: - geometry = {'coordinates': None, 'type': 'Polygon'} - - umm = item.get('umm') - - properties = { - 'beamModeType': get(umm, 'AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0), - 'browse': get(umm, 'RelatedUrls', ('Type', [('GET RELATED VISUALIZATION', 'URL')])), - 'bytes': cast(int, try_round_float(get(umm, 'AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0))), - 'centerLat': cast(float, get(umm, 'AdditionalAttributes', ('Name', 'CENTER_LAT'), 'Values', 0)), - 'centerLon': cast(float, get(umm, 'AdditionalAttributes', ('Name', 'CENTER_LON'), 'Values', 0)), - 'faradayRotation': cast(float, get(umm, 'AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0)), - 'fileID': get(umm, 'GranuleUR'), - 'flightDirection': get(umm, 'AdditionalAttributes', ('Name', 'ASCENDING_DESCENDING'), 'Values', 0), - 'groupID': get(umm, 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0), - 'granuleType': get(umm, 'AdditionalAttributes', ('Name', 'GRANULE_TYPE'), 'Values', 0), - 'insarStackId': get(umm, 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0), - 'md5sum': get(umm, 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0), - 'offNadirAngle': cast(float, get(umm, 'AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0)), - 'orbit': cast(int, get(umm, 'OrbitCalculatedSpatialDomains', 0, 'OrbitNumber')), - 'pathNumber': cast(int, get(umm, 'AdditionalAttributes', ('Name', 'PATH_NUMBER'), 'Values', 0)), - 'platform': get(umm, 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0), - 'pointingAngle': cast(float, get(umm, 'AdditionalAttributes', ('Name', 'POINTING_ANGLE'), 'Values', 0)), - 'polarization': get(umm, 'AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values', 0), - 'processingDate': get(umm, 'DataGranule', 'ProductionDateTime'), - 'processingLevel': get(umm, 'AdditionalAttributes', ('Name', 'PROCESSING_TYPE'), 'Values', 0), - 'sceneName': get(umm, 'DataGranule', 'Identifiers', ('IdentifierType', 'ProducerGranuleId'), 'Identifier'), - 'sensor': get(umm, 'Platforms', 0, 'Instruments', 0, 'ShortName'), - 'startTime': get(umm, 'TemporalExtent', 'RangeDateTime', 'BeginningDateTime'), - 'stopTime': get(umm, 'TemporalExtent', 'RangeDateTime', 'EndingDateTime'), - 'url': get(umm, 'RelatedUrls', ('Type', 'GET DATA'), 'URL'), - 'pgeVersion': get(umm, 'PGEVersionClass', 'PGEVersion') - } - - if properties['beamModeType'] is None: - properties['beamModeType'] = get(umm, 'AdditionalAttributes', ('Name', 'BEAM_MODE'), 'Values', 0) - - positions = {} - velocities = {} - positions['prePosition'], positions['prePositionTime'] = cast(get_state_vector, get(umm, 'AdditionalAttributes', ('Name', 'SV_POSITION_PRE'), 'Values', 0)) - positions['postPosition'], positions['postPositionTime'] = cast(get_state_vector, get(umm, 'AdditionalAttributes', ('Name', 'SV_POSITION_POST'), 'Values', 0)) - velocities['preVelocity'], velocities['preVelocityTime'] = cast(get_state_vector, get(umm, 'AdditionalAttributes', ('Name', 'SV_VELOCITY_PRE'), 'Values', 0)) - velocities['postVelocity'], velocities['postVelocityTime'] = cast(get_state_vector, get(umm, 'AdditionalAttributes', ('Name', 'SV_VELOCITY_POST'), 'Values', 0)) - ascendingNodeTime = get(umm, 'AdditionalAttributes', ('Name', 'ASC_NODE_TIME'), 'Values', 0) - - for key in ['prePositionTime','postPositionTime','preVelocityTime','postVelocityTime']: - if positions.get(key) is not None: - if not positions.get(key).endswith('Z'): - positions[key] += 'Z' - - if ascendingNodeTime is not None: - if not ascendingNodeTime.endswith('Z'): - ascendingNodeTime += 'Z' - - stateVectors = { - 'positions': positions, - 'velocities': velocities - } - - insarBaseline = cast(float, get(umm, 'AdditionalAttributes', ('Name', 'INSAR_BASELINE'), 'Values', 0)) - - baseline = {} - if None not in stateVectors['positions'].values() and len(stateVectors.items()) > 0: - baseline['stateVectors'] = stateVectors - baseline['ascendingNodeTime'] = ascendingNodeTime - elif insarBaseline is not None: - baseline['insarBaseline'] = insarBaseline - else: - baseline = None - - if properties['url'] is not None: - properties['fileName'] = properties['url'].split('/')[-1] - else: - properties['fileName'] = None - - if properties['platform'] is None: - properties['platform'] = get(umm, 'Platforms', 0, 'ShortName') - - asf_frame_platforms = ['Sentinel-1A', 'Sentinel-1B', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B', 'Sentinel-1 Interferogram', 'Sentinel-1 Interferogram (BETA)', 'ERS-1', 'ERS-2', 'JERS-1', 'RADARSAT-1'] - if properties['platform'] in asf_frame_platforms: - properties['frameNumber'] = cast(int, get(umm, 'AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0)) - else: - properties['frameNumber'] = cast(int, get(umm, 'AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0)) - - if properties['processingLevel'] == 'BURST': - burst = { - 'absoluteBurstID': cast(int, get(umm, 'AdditionalAttributes', ('Name', 'BURST_ID_ABSOLUTE'), 'Values', 0)), - 'relativeBurstID': cast(int, get(umm, 'AdditionalAttributes', ('Name', 'BURST_ID_RELATIVE'), 'Values', 0)), - 'fullBurstID': get(umm, 'AdditionalAttributes', ('Name', 'BURST_ID_FULL'), 'Values', 0), - 'burstIndex': cast(int, get(umm, 'AdditionalAttributes', ('Name', 'BURST_INDEX'), 'Values', 0)), - 'samplesPerBurst': cast(int, get(umm, 'AdditionalAttributes', ('Name', 'SAMPLES_PER_BURST'), 'Values', 0)), - 'subswath': get(umm, 'AdditionalAttributes', ('Name', 'SUBSWATH_NAME'), 'Values', 0), - 'azimuthTime': get(umm, 'AdditionalAttributes', ('Name', 'AZIMUTH_TIME'), 'Values', 0), - 'azimuthAnxTime': get(umm, 'AdditionalAttributes', ('Name', 'AZIMUTH_ANX_TIME'), 'Values', 0), - } - properties['burst'] = burst - properties['sceneName'] = properties['fileID'] - properties['bytes'] = cast(int, get(umm, 'AdditionalAttributes', ('Name', 'BYTE_LENGTH'), 'Values', 0)) - - urls = get(umm, 'RelatedUrls', ('Type', [('USE SERVICE API', 'URL')]), 0) - if urls is not None: - properties['url'] = urls[0] - properties['fileName'] = properties['fileID'] + '.' + urls[0].split('.')[-1] - properties['additionalUrls'] = [urls[1]] - - - if (fileID:=properties.get('fileID')): - if fileID.startswith('OPERA'): - properties['beamMode'] = get(umm, 'AdditionalAttributes', ('Name', 'BEAM_MODE'), 'Values', 0) - accessUrls = [*get(umm, 'RelatedUrls', ('Type', [('GET DATA', 'URL')]), 0), *get(umm, 'RelatedUrls', ('Type', [('EXTENDED METADATA', 'URL')]), 0)] - properties['additionalUrls'] = sorted([url for url in list(set(accessUrls)) if not url.endswith('.md5') - and not url.startswith('s3://') - and not 's3credentials' in url - and not url.endswith('.png') - and url != properties['url']]) - properties['polarization'] = get(umm, 'AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values') - - properties['operaBurstID'] = get(umm, 'AdditionalAttributes', ('Name', 'OPERA_BURST_ID'), 'Values', 0) - - if validityStartDate := get(umm, 'TemporalExtent', 'SingleDateTime'): - properties['validityStartDate'] = validityStartDate - - return {'geometry': geometry, 'properties': properties, 'type': 'Feature', 'baseline': baseline} - -def get_additional_fields(umm, *field_path): - return get(umm, *field_path) - -def cast(f, v): - try: - return f(v) - except TypeError: +# some products don't have integer values in BYTES fields, round to nearest int +def try_round_float(value: str) -> Optional[int]: + if value is None: return None + + value = float(value) + return round(value) - -def get(item: dict, *args): - if item is None: +def try_parse_int(value: str) -> Optional[int]: + if value is None: return None - for key in args: - if isinstance(key, int): - item = item[key] if key < len(item) else None - elif isinstance(key, tuple): - (a, b) = key - if isinstance(b, List): - output = [] - b = b[0] - for child in item: - if get(child, key[0]) == b[0]: - output.append(get(child, b[1])) - if len(output): - return output - - return None - - found = False - for child in item: - if get(child, a) == b: - item = child - found = True - break - if not found: - return None - else: - item = item.get(key) - if item is None: - return None - if item in [None, 'NA', 'N/A', '']: - item = None - return item - - -def get_state_vector(state_vector: str): - if state_vector is None: - return None, None - return list(map(float, state_vector.split(',')[:3])), state_vector.split(',')[-1] + return int(value) - -# some products don't have integer values in BYTES fields, round to nearest int -def try_round_float(value: str): - if value is not None: - value = float(value) - return round(value) +def try_parse_float(value: str) -> Optional[float]: + if value is None: + return None - return value - + return float(value) def fix_date(fixed_params: Dict[str, Any]): if 'start' in fixed_params or 'end' in fixed_params or 'season' in fixed_params: diff --git a/asf_search/Products/AIRSARProduct.py b/asf_search/Products/AIRSARProduct.py new file mode 100644 index 00000000..54c2c03c --- /dev/null +++ b/asf_search/Products/AIRSARProduct.py @@ -0,0 +1,25 @@ +import copy +from typing import Dict +from asf_search import ASFSession, ASFProduct +from asf_search.CMR.translate import try_parse_float, try_parse_int + +class AIRSARProduct(ASFProduct): + """ + ASF Dataset Overview Page: https://asf.alaska.edu/data-sets/sar-data-sets/airsar/ + """ + _base_properties = { + 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0], 'cast': try_parse_int}, + 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, + 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFProduct.get_property_paths(), + **AIRSARProduct._base_properties + } diff --git a/asf_search/Products/ALOSProduct.py b/asf_search/Products/ALOSProduct.py new file mode 100644 index 00000000..9f31011b --- /dev/null +++ b/asf_search/Products/ALOSProduct.py @@ -0,0 +1,39 @@ +from typing import Dict, Union +from asf_search import ASFSession, ASFProduct, ASFStackableProduct, ASFSearchOptions +from asf_search.CMR.translate import try_parse_float, try_parse_int, try_round_float +from asf_search.constants import PRODUCT_TYPE + + +class ALOSProduct(ASFStackableProduct): + """ + Used for ALOS Palsar and Avnir dataset products + + ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/alos-palsar/ + """ + _base_properties = { + 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0], 'cast': try_parse_int}, + 'faradayRotation': {'path': ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0], 'cast': try_parse_float}, + 'offNadirAngle': {'path': ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0], 'cast': try_parse_float}, + 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, + 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + if self.properties.get('groupID') is None: + self.properties['groupID'] = self.properties['sceneName'] + + @staticmethod + def get_default_baseline_product_type() -> Union[str, None]: + """ + Returns the product type to search for when building a baseline stack. + """ + return PRODUCT_TYPE.L1_1 + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFStackableProduct.get_property_paths(), + **ALOSProduct._base_properties + } diff --git a/asf_search/Products/ARIAS1GUNWProduct.py b/asf_search/Products/ARIAS1GUNWProduct.py new file mode 100644 index 00000000..2d88419a --- /dev/null +++ b/asf_search/Products/ARIAS1GUNWProduct.py @@ -0,0 +1,53 @@ +from typing import Dict +from asf_search import ASFSession +from asf_search.ASFSearchOptions import ASFSearchOptions +from asf_search.Products import S1Product +from asf_search.CMR.translate import try_parse_float + + +class ARIAS1GUNWProduct(S1Product): + """ + Used for ARIA S1 GUNW Products + + ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/derived-data-sets/sentinel-1-interferograms/ + """ + _base_properties = { + 'perpendicularBaseline': {'path': ['AdditionalAttributes', ('Name', 'PERPENDICULAR_BASELINE'), 'Values', 0], 'cast': try_parse_float}, + 'orbit': {'path': ['OrbitCalculatedSpatialDomains']} + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + self.properties['orbit'] = [orbit['OrbitNumber'] for orbit in self.properties['orbit']] + + urls = self.umm_get(self.umm, 'RelatedUrls', ('Type', [('USE SERVICE API', 'URL')]), 0) + if urls is not None: + self.properties['url'] = urls[0] + self.properties['fileName'] = self.properties['fileID'] + '.' + urls[0].split('.')[-1] + self.properties['additionalUrls'] = [urls[1]] + + @staticmethod + def get_property_paths() -> Dict: + return { + **S1Product.get_property_paths(), + **ARIAS1GUNWProduct._base_properties + } + + def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: + """ + Build search options that can be used to find an insar stack for this product + + :return: ASFSearchOptions describing appropriate options for building a stack from this product + """ + return None + + + def is_valid_reference(self): + return False + + @staticmethod + def get_default_baseline_product_type() -> None: + """ + Returns the product type to search for when building a baseline stack. + """ + return None \ No newline at end of file diff --git a/asf_search/Products/ERSProduct.py b/asf_search/Products/ERSProduct.py new file mode 100644 index 00000000..a2dbff98 --- /dev/null +++ b/asf_search/Products/ERSProduct.py @@ -0,0 +1,38 @@ +from typing import Dict, Union +from asf_search import ASFSearchOptions, ASFSession, ASFProduct, ASFStackableProduct +from asf_search.CMR.translate import try_round_float +from asf_search.constants import PRODUCT_TYPE + + +class ERSProduct(ASFStackableProduct): + """ + Used for ERS-1 and ERS-2 products + + ASF ERS-1 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-1/ + ASF ERS-2 Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/ers-2/ + """ + _base_properties = { + 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0]}, + 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, + 'esaFrame': {'path': ['AdditionalAttributes', ('Name', 'CENTER_ESA_FRAME'), 'Values', 0]}, + 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, + 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFStackableProduct.get_property_paths(), + **ERSProduct._base_properties + } + + @staticmethod + def get_default_baseline_product_type() -> Union[str, None]: + """ + Returns the product type to search for when building a baseline stack. + """ + return PRODUCT_TYPE.L0 diff --git a/asf_search/Products/JERSProduct.py b/asf_search/Products/JERSProduct.py new file mode 100644 index 00000000..1963225f --- /dev/null +++ b/asf_search/Products/JERSProduct.py @@ -0,0 +1,33 @@ +from typing import Dict, Union +from asf_search import ASFSearchOptions, ASFSession, ASFProduct, ASFStackableProduct +from asf_search.constants import PRODUCT_TYPE + + +class JERSProduct(ASFStackableProduct): + """ + ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/jers-1/ + """ + _base_properties = { + 'browse': {'path': ['RelatedUrls', ('Type', [('GET RELATED VISUALIZATION', 'URL')])]}, + 'groupID': {'path': ['AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, + 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, + 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + @staticmethod + def get_default_baseline_product_type() -> Union[str, None]: + """ + Returns the product type to search for when building a baseline stack. + """ + return PRODUCT_TYPE.L0 + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFStackableProduct.get_property_paths(), + **JERSProduct._base_properties + } diff --git a/asf_search/Products/OPERAS1Product.py b/asf_search/Products/OPERAS1Product.py new file mode 100644 index 00000000..0e3e1676 --- /dev/null +++ b/asf_search/Products/OPERAS1Product.py @@ -0,0 +1,92 @@ +from typing import Dict, Optional +from asf_search import ASFSearchOptions, ASFSession +from asf_search.Products import S1Product + + +class OPERAS1Product(S1Product): + """ + ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/opera/ + """ + _base_properties = { + 'centerLat': {'path': []}, # Opera products lacks these fields + 'centerLon': {'path': []}, + 'frameNumber': {'path': []}, + 'operaBurstID': {'path': ['AdditionalAttributes', ('Name', 'OPERA_BURST_ID'), 'Values', 0]}, + 'validityStartDate': {'path': ['TemporalExtent', 'SingleDateTime']}, + 'bytes': {'path': ['DataGranule', 'ArchiveAndDistributionInformation']}, + 'subswath': {'path': ['AdditionalAttributes', ('Name', 'SUBSWATH_NAME'), 'Values', 0]}, + 'polarization': {'path': ['AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values']} # dual polarization is in list rather than a 'VV+VH' style format + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + self.baseline = None + + self.properties['beamMode'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'BEAM_MODE'), 'Values', 0) + + accessUrls = [] + + if related_data_urls := self.umm_get(self.umm, 'RelatedUrls', ('Type', [('GET DATA', 'URL')]), 0): + accessUrls.extend(related_data_urls) + if related_metadata_urls := self.umm_get(self.umm, 'RelatedUrls', ('Type', [('EXTENDED METADATA', 'URL')]), 0): + accessUrls.extend(related_metadata_urls) + + self.properties['additionalUrls'] = sorted([ + url for url in list(set(accessUrls)) if not url.endswith('.md5') + and not url.startswith('s3://') + and 's3credentials' not in url + and not url.endswith('.png') + and url != self.properties['url'] + ]) + + self.properties['operaBurstID'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'OPERA_BURST_ID'), 'Values', 0) + self.properties['bytes'] = {entry['Name']: {'bytes': entry['SizeInBytes'], 'format': entry['Format']} for entry in self.properties['bytes']} + + center = self.centroid() + self.properties['centerLat'] = center.y + self.properties['centerLon'] = center.x + + self.properties.pop('frameNumber') + + if (processingLevel := self.properties['processingLevel']) in ['RTC', 'RTC-STATIC']: + self.properties['bistaticDelayCorrection'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'BISTATIC_DELAY_CORRECTION'), 'Values', 0) + if processingLevel == 'RTC': + self.properties['noiseCorrection'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'NOISE_CORRECTION'), 'Values', 0) + self.properties['postProcessingFilter'] = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'POST_PROCESSING_FILTER'), 'Values', 0) + + def get_stack_opts(self, opts: ASFSearchOptions = ASFSearchOptions()) -> ASFSearchOptions: + return opts + + @staticmethod + def get_property_paths() -> Dict: + return { + **S1Product.get_property_paths(), + **OPERAS1Product._base_properties + } + + @staticmethod + def get_default_baseline_product_type() -> None: + """ + Returns the product type to search for when building a baseline stack. + """ + return None + + def is_valid_reference(self): + return False + + def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: + """ + Build search options that can be used to find an insar stack for this product + + :return: ASFSearchOptions describing appropriate options for building a stack from this product + """ + return None + + def get_sort_keys(self): + keys = super().get_sort_keys() + + if keys[0] is None: + keys = self.properties.get('validityStartDate'), keys[1] + + return keys diff --git a/asf_search/Products/RADARSATProduct.py b/asf_search/Products/RADARSATProduct.py new file mode 100644 index 00000000..7db7f1b2 --- /dev/null +++ b/asf_search/Products/RADARSATProduct.py @@ -0,0 +1,33 @@ +from typing import Dict, Union +from asf_search import ASFSearchOptions, ASFSession, ASFProduct, ASFStackableProduct +from asf_search.CMR.translate import try_parse_float +from asf_search.constants import PRODUCT_TYPE + + +class RADARSATProduct(ASFStackableProduct): + """ + ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/radarsat-1/ + """ + _base_properties = { + 'faradayRotation': {'path': ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0], 'cast': try_parse_float}, + 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, + 'insarStackId': {'path': ['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFStackableProduct.get_property_paths(), + **RADARSATProduct._base_properties + } + + @staticmethod + def get_default_baseline_product_type() -> Union[str, None]: + """ + Returns the product type to search for when building a baseline stack. + """ + return PRODUCT_TYPE.L0 diff --git a/asf_search/Products/S1BurstProduct.py b/asf_search/Products/S1BurstProduct.py new file mode 100644 index 00000000..9bd59662 --- /dev/null +++ b/asf_search/Products/S1BurstProduct.py @@ -0,0 +1,90 @@ +import copy +from typing import Dict, Union +from asf_search import ASFSearchOptions, ASFSession +from asf_search.Products import S1Product +from asf_search.CMR.translate import try_parse_int +from asf_search.constants import PRODUCT_TYPE + +class S1BurstProduct(S1Product): + """ + S1Product Subclass made specifically for Sentinel-1 SLC-BURST products + + Key features/properties: + - `properties['burst']` contains SLC-BURST Specific fields such as `fullBurstID` and `burstIndex` + - `properties['additionalUrls']` contains BURST-XML url + - SLC-BURST specific stacking params + + ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/data-sets/derived-data-sets/sentinel-1-bursts/ + """ + _base_properties = { + 'bytes': {'path': ['AdditionalAttributes', ('Name', 'BYTE_LENGTH'), 'Values', 0]}, + 'absoluteBurstID': {'path': ['AdditionalAttributes', ('Name', 'BURST_ID_ABSOLUTE'), 'Values', 0], 'cast': try_parse_int}, + 'relativeBurstID': {'path': ['AdditionalAttributes', ('Name', 'BURST_ID_RELATIVE'), 'Values', 0], 'cast': try_parse_int}, + 'fullBurstID': {'path': ['AdditionalAttributes', ('Name', 'BURST_ID_FULL'), 'Values', 0]}, + 'burstIndex': {'path': ['AdditionalAttributes', ('Name', 'BURST_INDEX'), 'Values', 0], 'cast': try_parse_int}, + 'samplesPerBurst': {'path': ['AdditionalAttributes', ('Name', 'SAMPLES_PER_BURST'), 'Values', 0], 'cast': try_parse_int}, + 'subswath': {'path': ['AdditionalAttributes', ('Name', 'SUBSWATH_NAME'), 'Values', 0]}, + 'azimuthTime': {'path': ['AdditionalAttributes', ('Name', 'AZIMUTH_TIME'), 'Values', 0]}, + 'azimuthAnxTime': {'path': ['AdditionalAttributes', ('Name', 'AZIMUTH_ANX_TIME'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + self.properties['sceneName'] = self.properties['fileID'] + + # Gathers burst properties into `burst` specific dict + # rather than properties dict to limit breaking changes + self.properties['burst'] = { + 'absoluteBurstID': self.properties.pop('absoluteBurstID'), + 'relativeBurstID': self.properties.pop('relativeBurstID'), + 'fullBurstID': self.properties.pop('fullBurstID'), + 'burstIndex': self.properties.pop('burstIndex'), + 'samplesPerBurst': self.properties.pop('samplesPerBurst'), + 'subswath': self.properties.pop('subswath'), + 'azimuthTime': self.properties.pop('azimuthTime'), + 'azimuthAnxTime': self.properties.pop('azimuthAnxTime') + } + + urls = self.umm_get(self.umm, 'RelatedUrls', ('Type', [('USE SERVICE API', 'URL')]), 0) + if urls is not None: + self.properties['url'] = urls[0] + self.properties['fileName'] = self.properties['fileID'] + '.' + urls[0].split('.')[-1] + self.properties['additionalUrls'] = [urls[1]] # xml-metadata url + + def get_stack_opts(self, opts: ASFSearchOptions = None): + """ + Returns the search options asf-search will use internally to build an SLC-BURST baseline stack from + + :param opts: additional criteria for limiting + :returns ASFSearchOptions used for build Sentinel-1 SLC-BURST Stack + """ + stack_opts = (ASFSearchOptions() if opts is None else copy(opts)) + + stack_opts.processingLevel = self.get_default_baseline_product_type() + stack_opts.fullBurstID = self.properties['burst']['fullBurstID'] + stack_opts.polarization = [self.properties['polarization']] + return stack_opts + + @staticmethod + def get_property_paths() -> Dict: + return { + **S1Product.get_property_paths(), + **S1BurstProduct._base_properties + } + + def _get_additional_filenames_and_urls(self, default_filename: str = None): + # Burst XML filenames are just numbers, this makes it more indentifiable + if default_filename is None: + default_filename = self.properties['fileName'] + + file_name = f"{'.'.join(default_filename.split('.')[:-1])}.xml" + + return [(file_name, self.properties['additionalUrls'][0])] + + @staticmethod + def get_default_baseline_product_type() -> Union[str, None]: + """ + Returns the product type to search for when building a baseline stack. + """ + return PRODUCT_TYPE.BURST + \ No newline at end of file diff --git a/asf_search/Products/S1Product.py b/asf_search/Products/S1Product.py new file mode 100644 index 00000000..25282de7 --- /dev/null +++ b/asf_search/Products/S1Product.py @@ -0,0 +1,141 @@ +import copy +from typing import Dict, List, Optional, Tuple +from asf_search import ASFSearchOptions, ASFSession, ASFStackableProduct +from asf_search.CMR.translate import try_parse_int +from asf_search.constants import PLATFORM +from asf_search.constants import PRODUCT_TYPE + + +class S1Product(ASFStackableProduct): + """ + The S1Product classes covers most Sentinel-1 Products + (For S1 BURST-SLC, OPERA-S1, and ARIA-S1 GUNW Products, see relevant S1 subclasses) + + ASF Dataset Overview Page: https://asf.alaska.edu/datasets/daac/sentinel-1/ + """ + + _base_properties = { + 'frameNumber': {'path': ['AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0], 'cast': try_parse_int}, #Sentinel and ALOS product alt for frameNumber (ESA_FRAME) + 'groupID': {'path': ['AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, + 'md5sum': {'path': ['AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + 'pgeVersion': {'path': ['PGEVersionClass', 'PGEVersion']}, + } + """ + S1 Specific path override + - frameNumber: overrides ASFProduct's `CENTER_ESA_FRAME` with `FRAME_NUMBER` + """ + + baseline_type = ASFStackableProduct.BaselineCalcType.CALCULATED + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + if self._has_baseline(): + self.baseline = self.get_baseline_calc_properties() + + def _has_baseline(self) -> bool: + baseline = self.get_baseline_calc_properties() + + return ( + baseline is not None and + None not in baseline['stateVectors']['positions'].values() + ) + + def get_baseline_calc_properties(self) -> Dict: + """ + :returns properties required for SLC baseline stack calculations + """ + ascendingNodeTime = self.umm_cast( + self._parse_timestamp, + self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'ASC_NODE_TIME'), 'Values', 0) + ) + + return { + 'stateVectors': self.get_state_vectors(), + 'ascendingNodeTime': ascendingNodeTime + } + + def get_state_vectors(self) -> Dict: + """ + Used in spatio-temporal perpendicular baseline calculations for non-pre-calculated stacks + + :returns dictionary of pre/post positions, velocities, and times""" + positions = {} + velocities = {} + + sv_pre_position = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'SV_POSITION_PRE'), 'Values', 0) + sv_post_position = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'SV_POSITION_POST'), 'Values', 0) + sv_pre_velocity = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'SV_VELOCITY_PRE'), 'Values', 0) + sv_post_velocity = self.umm_get(self.umm, 'AdditionalAttributes', ('Name', 'SV_VELOCITY_POST'), 'Values', 0) + + positions['prePosition'], positions['prePositionTime'] = self.umm_cast(self._parse_state_vector, sv_pre_position) + positions['postPosition'], positions['postPositionTime'] = self.umm_cast(self._parse_state_vector, sv_post_position) + velocities['preVelocity'], velocities['preVelocityTime'] = self.umm_cast(self._parse_state_vector, sv_pre_velocity) + velocities['postVelocity'], velocities['postVelocityTime'] = self.umm_cast(self._parse_state_vector, sv_post_velocity) + + return { + 'positions': positions, + 'velocities': velocities + } + + def _parse_timestamp(self, timestamp: str) -> Optional[str]: + if timestamp is None: + return None + + return timestamp if timestamp.endswith('Z') else f'{timestamp}Z' + + def _parse_state_vector(self, state_vector: str) -> Tuple[Optional[List], Optional[str]]: + if state_vector is None: + return None, None + + velocity = [float(val) for val in state_vector.split(',')[:3]] + timestamp = self._parse_timestamp(state_vector.split(',')[-1]) + + return velocity, timestamp + + def get_stack_opts(self, opts: ASFSearchOptions = None) -> ASFSearchOptions: + """ + Returns the search options asf-search will use internally to build an SLC baseline stack from + + :param opts: additional criteria for limiting + :returns ASFSearchOptions used for build Sentinel-1 SLC Stack + """ + stack_opts = (ASFSearchOptions() if opts is None else copy(opts)) + + stack_opts.processingLevel = self.get_default_baseline_product_type() + stack_opts.beamMode = [self.properties['beamModeType']] + stack_opts.flightDirection = self.properties['flightDirection'] + stack_opts.relativeOrbit = [int(self.properties['pathNumber'])] # path + stack_opts.platform = [PLATFORM.SENTINEL1A, PLATFORM.SENTINEL1B] + + if self.properties['polarization'] in ['HH', 'HH+HV']: + stack_opts.polarization = ['HH', 'HH+HV'] + else: + stack_opts.polarization = ['VV', 'VV+VH'] + + stack_opts.intersectsWith = self.centroid().wkt + + return stack_opts + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFStackableProduct.get_property_paths(), + **S1Product._base_properties + } + + def is_valid_reference(self) -> bool: + keys = ['postPosition', 'postPositionTime', 'prePosition', 'postPositionTime'] + + for key in keys: + if self.baseline['stateVectors']['positions'].get(key) is None: + return False + + return True + + @staticmethod + def get_default_baseline_product_type() -> str: + """ + Returns the product type to search for when building a baseline stack. + """ + return PRODUCT_TYPE.SLC diff --git a/asf_search/Products/SEASATProduct.py b/asf_search/Products/SEASATProduct.py new file mode 100644 index 00000000..e726d756 --- /dev/null +++ b/asf_search/Products/SEASATProduct.py @@ -0,0 +1,24 @@ +from typing import Dict +from asf_search import ASFSession, ASFProduct +from asf_search.CMR.translate import try_parse_float, try_round_float + + +class SEASATProduct(ASFProduct): + """ + ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/sar-data-sets/seasat/ + """ + _base_properties = { + 'bytes': {'path': [ 'AdditionalAttributes', ('Name', 'BYTES'), 'Values', 0], 'cast': try_round_float}, + 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFProduct.get_property_paths(), + **SEASATProduct._base_properties + } diff --git a/asf_search/Products/SIRCProduct.py b/asf_search/Products/SIRCProduct.py new file mode 100644 index 00000000..e5e9ad31 --- /dev/null +++ b/asf_search/Products/SIRCProduct.py @@ -0,0 +1,23 @@ +from typing import Dict +from asf_search import ASFProduct, ASFSession + +class SIRCProduct(ASFProduct): + """ + Dataset Documentation Page: https://eospso.nasa.gov/missions/spaceborne-imaging-radar-c + """ + _base_properties = { + 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, + 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + 'pgeVersion': {'path': ['PGEVersionClass', 'PGEVersion'] }, + 'beamModeType': {'path': ['AdditionalAttributes', ('Name', 'BEAM_MODE_TYPE'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFProduct.get_property_paths(), + **SIRCProduct._base_properties + } diff --git a/asf_search/Products/SMAPProduct.py b/asf_search/Products/SMAPProduct.py new file mode 100644 index 00000000..f78f00e0 --- /dev/null +++ b/asf_search/Products/SMAPProduct.py @@ -0,0 +1,24 @@ +import copy +from typing import Dict +from asf_search import ASFProduct, ASFSession +from asf_search.CMR.translate import try_parse_float + +class SMAPProduct(ASFProduct): + """ + ASF Dataset Documentation Page: https://asf.alaska.edu/data-sets/sar-data-sets/soil-moisture-active-passive-smap-mission/ + """ + _base_properties = { + 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, + 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFProduct.get_property_paths(), + **SMAPProduct._base_properties + } diff --git a/asf_search/Products/UAVSARProduct.py b/asf_search/Products/UAVSARProduct.py new file mode 100644 index 00000000..73acd812 --- /dev/null +++ b/asf_search/Products/UAVSARProduct.py @@ -0,0 +1,24 @@ +import copy +from typing import Dict +from asf_search import ASFProduct, ASFSession +from asf_search.CMR.translate import try_parse_float + +class UAVSARProduct(ASFProduct): + """ + ASF Dataset Documentation Page: https://asf.alaska.edu/datasets/daac/uavsar/ + """ + _base_properties = { + 'groupID': {'path': [ 'AdditionalAttributes', ('Name', 'GROUP_ID'), 'Values', 0]}, + 'insarStackId': {'path': [ 'AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]}, + 'md5sum': {'path': [ 'AdditionalAttributes', ('Name', 'MD5SUM'), 'Values', 0]}, + } + + def __init__(self, args: Dict = {}, session: ASFSession = ASFSession()): + super().__init__(args, session) + + @staticmethod + def get_property_paths() -> Dict: + return { + **ASFProduct.get_property_paths(), + **UAVSARProduct._base_properties + } diff --git a/asf_search/Products/__init__.py b/asf_search/Products/__init__.py new file mode 100644 index 00000000..dbdad6fe --- /dev/null +++ b/asf_search/Products/__init__.py @@ -0,0 +1,13 @@ +from .S1Product import S1Product +from .ALOSProduct import ALOSProduct +from .RADARSATProduct import RADARSATProduct +from .AIRSARProduct import AIRSARProduct +from .ERSProduct import ERSProduct +from .JERSProduct import JERSProduct +from .UAVSARProduct import UAVSARProduct +from .SIRCProduct import SIRCProduct +from .SEASATProduct import SEASATProduct +from .SMAPProduct import SMAPProduct +from .S1BurstProduct import S1BurstProduct +from .OPERAS1Product import OPERAS1Product +from .ARIAS1GUNWProduct import ARIAS1GUNWProduct diff --git a/asf_search/__init__.py b/asf_search/__init__.py index da317d1e..4cc55396 100644 --- a/asf_search/__init__.py +++ b/asf_search/__init__.py @@ -21,9 +21,12 @@ from .ASFSession import ASFSession from .ASFProduct import ASFProduct +from .ASFStackableProduct import ASFStackableProduct from .ASFSearchResults import ASFSearchResults from .ASFSearchOptions import ASFSearchOptions, validators -from .constants import * +from .Products import * +from .exceptions import * +from .constants import BEAMMODE, FLIGHT_DIRECTION, INSTRUMENT, PLATFORM, POLARIZATION, PRODUCT_TYPE, INTERNAL, DATASET from .exceptions import * from .health import * from .search import * @@ -33,5 +36,5 @@ from .WKT import validate_wkt from .export import * -REPORT_ERRORS=True +REPORT_ERRORS=True """Enables automatic search error reporting to ASF, send any questions to uso@asf.alaska.edu""" diff --git a/asf_search/baseline/__init__.py b/asf_search/baseline/__init__.py index 65a9d9e6..57ecb405 100644 --- a/asf_search/baseline/__init__.py +++ b/asf_search/baseline/__init__.py @@ -1,2 +1,2 @@ from .calc import * -from .stack import * +from .stack import * \ No newline at end of file diff --git a/asf_search/baseline/stack.py b/asf_search/baseline/stack.py index 638917ee..69b66f85 100644 --- a/asf_search/baseline/stack.py +++ b/asf_search/baseline/stack.py @@ -1,74 +1,62 @@ +from typing import Tuple, List from dateutil.parser import parse import pytz + from .calc import calculate_perpendicular_baselines -from asf_search import ASFProduct, ASFSearchResults +from asf_search import ASFProduct, ASFStackableProduct, ASFSearchResults -precalc_datasets = ['AL', 'R1', 'E1', 'E2', 'J1'] -def get_baseline_from_stack(reference: ASFProduct, stack: ASFSearchResults): - warnings = None +def get_baseline_from_stack(reference: ASFProduct, stack: ASFSearchResults) -> Tuple[ASFSearchResults, List[dict]]: + warnings = [] if len(stack) == 0: raise ValueError('No products found matching stack parameters') - stack = [product for product in stack if not product.properties['processingLevel'].lower().startswith('metadata') and product.baseline != None] - reference, stack, warnings = check_reference(reference, stack) + + stack = [product for product in stack if not product.properties['processingLevel'].lower().startswith('metadata') and product.baseline is not None] + reference, stack, reference_warnings = check_reference(reference, stack) + if reference_warnings is not None: + warnings.append(reference_warnings) + + stack = calculate_temporal_baselines(reference, stack) - if get_platform(reference.properties['sceneName']) in precalc_datasets: + if reference.baseline_type == ASFStackableProduct.BaselineCalcType.PRE_CALCULATED: stack = offset_perpendicular_baselines(reference, stack) else: stack = calculate_perpendicular_baselines(reference.properties['sceneName'], stack) + missing_state_vectors = _count_missing_state_vectors(stack) + if missing_state_vectors > 0: + warnings.append({'MISSING STATE VECTORS': f'{missing_state_vectors} scenes in stack missing State Vectors, perpendicular baseline not calculated for these scenes'}) + return ASFSearchResults(stack), warnings - -def valid_state_vectors(product: ASFProduct): - if product is None: - raise ValueError('Attempting to check state vectors on None, this is fatal') - for key in ['postPosition', 'postPositionTime', 'prePosition', 'postPositionTime']: - if key not in product.baseline['stateVectors']['positions'] or product.baseline['stateVectors']['positions'][key] == None: - return False - return True + +def _count_missing_state_vectors(stack) -> int: + return len([scene for scene in stack if scene.baseline.get('noStateVectors')]) def find_new_reference(stack: ASFSearchResults): for product in stack: - if valid_state_vectors(product): + if product.is_valid_reference(): return product return None + def check_reference(reference: ASFProduct, stack: ASFSearchResults): warnings = None if reference.properties['sceneName'] not in [product.properties['sceneName'] for product in stack]: # Somehow the reference we built the stack from is missing?! Just pick one reference = stack[0] warnings = [{'NEW_REFERENCE': 'A new reference scene had to be selected in order to calculate baseline values.'}] - if get_platform(reference.properties['sceneName']) in precalc_datasets: - if 'insarBaseline' not in reference.baseline: - raise ValueError('No baseline values available for precalculated dataset') - else: - if not valid_state_vectors(reference): # the reference might be missing state vectors, pick a valid reference, replace above warning if it also happened - reference = find_new_reference(stack) - if reference == None: - raise ValueError('No valid state vectors on any scenes in stack, this is fatal') - warnings = [{'NEW_REFERENCE': 'A new reference had to be selected in order to calculate baseline values.'}] + # non-s1 is_valid_reference raise an error, while we try to find a valid s1 reference + # do we want this behaviour for pre-calc stacks? + if not reference.is_valid_reference(): + reference = find_new_reference(stack) + if reference == None: + raise ValueError('No valid state vectors on any scenes in stack, this is fatal') return reference, stack, warnings -def get_platform(reference: str): - return reference[0:2].upper() - -def get_default_product_type(product: ASFProduct): - scene_name = product.properties['sceneName'] - - if get_platform(scene_name) in ['AL']: - return 'L1.1' - if get_platform(scene_name) in ['R1', 'E1', 'E2', 'J1']: - return 'L0' - if get_platform(scene_name) in ['S1']: - if product.properties['processingLevel'] == 'BURST': - return 'BURST' - return 'SLC' - return None def calculate_temporal_baselines(reference: ASFProduct, stack: ASFSearchResults): """ @@ -87,13 +75,13 @@ def calculate_temporal_baselines(reference: ASFProduct, stack: ASFSearchResults) if secondary_time.tzinfo is None: secondary_time = pytz.utc.localize(secondary_time) secondary.properties['temporalBaseline'] = (secondary_time.date() - reference_time.date()).days - + return stack def offset_perpendicular_baselines(reference: ASFProduct, stack: ASFSearchResults): reference_offset = float(reference.baseline['insarBaseline']) - + for product in stack: product.properties['perpendicularBaseline'] = round(float(product.baseline['insarBaseline']) - reference_offset) - + return stack diff --git a/asf_search/constants/DATASET.py b/asf_search/constants/DATASET.py index 2ccf52ea..2e4aeac4 100644 --- a/asf_search/constants/DATASET.py +++ b/asf_search/constants/DATASET.py @@ -1,5 +1,6 @@ SENTINEL1 = 'SENTINEL-1' OPERA_S1 = 'OPERA-S1' +OPERA_S1_CALIBRATION = 'OPERA-S1-CALIBRATION' SLC_BURST = 'SLC-BURST' ALOS_PALSAR = 'ALOS PALSAR' ALOS_AVNIR_2 = 'ALOS AVNIR-2' diff --git a/asf_search/download/download.py b/asf_search/download/download.py index 3de9fae3..a07ffff2 100644 --- a/asf_search/download/download.py +++ b/asf_search/download/download.py @@ -5,13 +5,16 @@ from requests import Response from requests.exceptions import HTTPError import warnings -import regex as re from asf_search.exceptions import ASFAuthenticationError, ASFDownloadError -from asf_search import ASFSession -from remotezip import RemoteZip +from asf_search import ASF_LOGGER, ASFSession from tenacity import retry, stop_after_delay, retry_if_result, wait_fixed +try: + from remotezip import RemoteZip +except ImportError: + RemoteZip = None + def _download_url(arg): url, path, session = arg download_url( @@ -74,12 +77,14 @@ def download_url(url: str, path: str, filename: str = None, session: ASFSession for chunk in response.iter_content(chunk_size=8192): f.write(chunk) -def remotezip(url: str, session: ASFSession) -> RemoteZip: +def remotezip(url: str, session: ASFSession) -> 'RemoteZip': """ :param url: the url to the zip product :param session: the authenticated ASFSession to read and download from the zip file """ - + if RemoteZip is None: + raise ImportError("Could not find remotezip package in current python environment. \"remotezip\" is an optional dependency of asf-search required for the `remotezip()` method. Enable by including the appropriate pip or conda install. Ex: `python3 -m pip install asf-search[extras]`") + session.hooks['response'].append(strip_auth_if_aws) return RemoteZip(url, session=session) diff --git a/asf_search/export/csv.py b/asf_search/export/csv.py index 32a1fb8b..575e7320 100644 --- a/asf_search/export/csv.py +++ b/asf_search/export/csv.py @@ -1,7 +1,6 @@ import csv from types import GeneratorType from asf_search import ASF_LOGGER -from asf_search.CMR.translate import get_additional_fields from asf_search.export.export_translators import ASFSearchResults_to_properties_list import inspect @@ -21,6 +20,7 @@ ('doppler', ['AdditionalAttributes', ('Name', 'DOPPLER'), 'Values', 0]), ('sizeMB', ['DataGranule', 'ArchiveAndDistributionInformation', 0, 'Size']), ('insarStackSize', ['AdditionalAttributes', ('Name', 'INSAR_STACK_SIZE'), 'Values', 0]), + ('offNadirAngle', ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0]) ] fieldnames = ( @@ -92,7 +92,7 @@ def get_additional_output_fields(self, product): additional_fields = {} for key, path in extra_csv_fields: - additional_fields[key] = get_additional_fields(umm, *path) + additional_fields[key] = product.umm_get(product.umm, *path) return additional_fields @@ -117,38 +117,38 @@ def streamRows(self): def getItem(self, p): return { - "Granule Name":p['sceneName'], - "Platform":p['platform'], - "Sensor":p['sensor'], - "Beam Mode":p['beamModeType'], - "Beam Mode Description":p['configurationName'], - "Orbit":p['orbit'], - "Path Number":p['pathNumber'], - "Frame Number":p['frameNumber'], - "Acquisition Date":p['sceneDate'], - "Processing Date":p['processingDate'], - "Processing Level":p['processingLevel'], - "Start Time":p['startTime'], - "End Time":p['stopTime'], - "Center Lat":p['centerLat'], - "Center Lon":p['centerLon'], - "Near Start Lat":p['nearStartLat'], - "Near Start Lon":p['nearStartLon'], - "Far Start Lat":p['farStartLat'], - "Far Start Lon":p['farStartLon'], - "Near End Lat":p['nearEndLat'], - "Near End Lon":p['nearEndLon'], - "Far End Lat":p['farEndLat'], - "Far End Lon":p['farEndLon'], - "Faraday Rotation":p['faradayRotation'], - "Ascending or Descending?":p['flightDirection'], - "URL":p['url'], - "Size (MB)":p['sizeMB'], - "Off Nadir Angle":p['offNadirAngle'], - "Stack Size":p['insarStackSize'], - "Doppler":p['doppler'], - "GroupID":p['groupID'], - "Pointing Angle":p['pointingAngle'], + "Granule Name":p.get('sceneName'), + "Platform":p.get('platform'), + "Sensor":p.get('sensor'), + "Beam Mode":p.get('beamModeType'), + "Beam Mode Description":p.get('configurationName'), + "Orbit":p.get('orbit'), + "Path Number":p.get('pathNumber'), + "Frame Number":p.get('frameNumber'), + "Acquisition Date":p.get('sceneDate'), + "Processing Date":p.get('processingDate'), + "Processing Level":p.get('processingLevel'), + "Start Time":p.get('startTime'), + "End Time":p.get('stopTime'), + "Center Lat":p.get('centerLat'), + "Center Lon":p.get('centerLon'), + "Near Start Lat":p.get('nearStartLat'), + "Near Start Lon":p.get('nearStartLon'), + "Far Start Lat":p.get('farStartLat'), + "Far Start Lon":p.get('farStartLon'), + "Near End Lat":p.get('nearEndLat'), + "Near End Lon":p.get('nearEndLon'), + "Far End Lat":p.get('farEndLat'), + "Far End Lon":p.get('farEndLon'), + "Faraday Rotation":p.get('faradayRotation'), + "Ascending or Descending?":p.get('flightDirection'), + "URL":p.get('url'), + "Size (MB)":p.get('sizeMB'), + "Off Nadir Angle":p.get('offNadirAngle'), + "Stack Size":p.get('insarStackSize'), + "Doppler":p.get('doppler'), + "GroupID":p.get('groupID'), + "Pointing Angle":p.get('pointingAngle'), "TemporalBaseline":p.get('teporalBaseline'), "PerpendicularBaseline":p.get('pependicularBaseline'), "relativeBurstID": p['burst']['relativeBurstID'] if p['processingLevel'] == 'BURST' else None, diff --git a/asf_search/export/jsonlite.py b/asf_search/export/jsonlite.py index dd697dab..8f581cfd 100644 --- a/asf_search/export/jsonlite.py +++ b/asf_search/export/jsonlite.py @@ -5,8 +5,7 @@ from shapely.geometry import shape from shapely.ops import transform -from asf_search.CMR.translate import get_additional_fields -from asf_search import ASF_LOGGER, ASFProduct +from asf_search import ASF_LOGGER from asf_search.export.export_translators import ASFSearchResults_to_properties_list extra_jsonlite_fields = [ @@ -60,19 +59,19 @@ def __iter__(self): def __len__(self): return self.len - def get_additional_output_fields(self, product: ASFProduct): - umm = product.umm + def get_additional_output_fields(self, product): + # umm = product.umm additional_fields = {} for key, path in extra_jsonlite_fields: - additional_fields[key] = get_additional_fields(umm, *path) + additional_fields[key] = product.umm_get(product.umm, *path) if product.properties['platform'].upper() in ['ALOS', 'RADARSAT-1', 'JERS-1', 'ERS-1', 'ERS-2']: - insarGrouping = get_additional_fields(umm, *['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]) + insarGrouping = product.umm_get(product.umm, *['AdditionalAttributes', ('Name', 'INSAR_STACK_ID'), 'Values', 0]) if insarGrouping not in [None, 0, '0', 'NA', 'NULL']: additional_fields['canInsar'] = True - additional_fields['insarStackSize'] = get_additional_fields(umm, *['AdditionalAttributes', ('Name', 'INSAR_STACK_SIZE'), 'Values', 0]) + additional_fields['insarStackSize'] = product.umm_get(product.umm, *['AdditionalAttributes', ('Name', 'INSAR_STACK_SIZE'), 'Values', 0]) else: additional_fields['canInsar'] = False else: @@ -101,19 +100,20 @@ def getItem(self, p): if p[i] == 'NA' or p[i] == '': p[i] = None try: - if float(p['offNadirAngle']) < 0: - p['offNadirAngle'] = None + if p.get('offNadirAngle') is not None and float(p['offNadirAngle']) < 0: + p['offNadirAngle'] = None except TypeError: pass try: - if float(p['pathNumber']) < 0: - p['pathNumber'] = None + if p.get('patNumber'): + if float(p['pathNumber']) < 0: + p['pathNumber'] = None except TypeError: pass try: - if p['groupID'] is None: + if p.get('groupID') is None: p['groupID'] = p['sceneName'] except TypeError: pass @@ -142,34 +142,34 @@ def getItem(self, p): result = { 'beamMode': p['beamModeType'], 'browse': [] if p.get('browse') is None else p.get('browse'), - 'canInSAR': p['canInsar'], - 'dataset': p['platform'], - 'downloadUrl': p['url'], - 'faradayRotation': p['faradayRotation'], # ALOS - 'fileName': p['fileName'], - 'flightDirection': p['flightDirection'], - 'flightLine': p['flightLine'], - 'frame': p['frameNumber'], - 'granuleName': p['sceneName'], - 'groupID': p['groupID'], - 'instrument': p['sensor'], - 'missionName': p['missionName'], - 'offNadirAngle': str(p['offNadirAngle']) if p['offNadirAngle'] is not None else None, # ALOS + 'canInSAR': p.get('canInsar'), + 'dataset': p.get('platform'), + 'downloadUrl': p.get('url'), + 'faradayRotation': p.get('faradayRotation'), # ALOS + 'fileName': p.get('fileName'), + 'flightDirection': p.get('flightDirection'), + 'flightLine': p.get('flightLine'), + 'frame': p.get('frameNumber'), + 'granuleName': p.get('sceneName'), + 'groupID': p.get('groupID'), + 'instrument': p.get('sensor'), + 'missionName': p.get('missionName'), + 'offNadirAngle': str(p['offNadirAngle']) if p.get('offNadirAngle') is not None else None, # ALOS 'orbit': [str(p['orbit'])], - 'path': p['pathNumber'], - 'polarization': p['polarization'], - 'pointingAngle': p['pointingAngle'], - 'productID': p['fileID'], - 'productType': p['processingLevel'], - 'productTypeDisplay': p['processingTypeDisplay'], - 'sizeMB': p['sizeMB'], + 'path': p.get('pathNumber'), + 'polarization': p.get('polarization'), + 'pointingAngle': p.get('pointingAngle'), + 'productID': p.get('fileID'), + 'productType': p.get('processingLevel'), + 'productTypeDisplay': p.get('processingTypeDisplay'), + 'sizeMB': p.get('sizeMB'), 'stackSize': p.get('insarStackSize'), # Used for datasets with precalculated stacks - 'startTime': p['startTime'], - 'stopTime': p['stopTime'], - 'thumb': p['thumb'], + 'startTime': p.get('startTime'), + 'stopTime': p.get('stopTime'), + 'thumb': p.get('thumb'), 'wkt': wrapped, 'wkt_unwrapped': unwrapped, - 'pgeVersion': p['pgeVersion'] + 'pgeVersion': p.get('pgeVersion') } for key in result.keys(): diff --git a/asf_search/export/kml.py b/asf_search/export/kml.py index c93c017f..57f6d638 100644 --- a/asf_search/export/kml.py +++ b/asf_search/export/kml.py @@ -1,7 +1,7 @@ import inspect from types import GeneratorType +from typing import Dict from asf_search import ASF_LOGGER -from asf_search.CMR import get_additional_fields from asf_search.export.metalink import MetalinkStreamArray import xml.etree.ElementTree as ETree @@ -13,6 +13,7 @@ ('shape', ['SpatialExtent', 'HorizontalSpatialDomain', 'Geometry', 'GPolygons', 0, 'Boundary', 'Points']), ('thumbnailUrl', ['AdditionalAttributes', ('Name', 'THUMBNAIL_URL'), 'Values', 0]), ('faradayRotation', ['AdditionalAttributes', ('Name', 'FARADAY_ROTATION'), 'Values', 0]), + ('offNadirAngle', ['AdditionalAttributes', ('Name', 'OFF_NADIR_ANGLE'), 'Values', 0]) ] def results_to_kml(results): @@ -41,16 +42,17 @@ def __init__(self, results): \n """ self.footer = """\n""" - + + + def getOutputType(self) -> str: + return 'kml' + def get_additional_fields(self, product): umm = product.umm additional_fields = {} for key, path in extra_kml_fields: - additional_fields[key] = get_additional_fields(umm, *path) + additional_fields[key] = product.umm_get(umm, *path) return additional_fields - - def getOutputType(self) -> str: - return 'kml' def getItem(self, p): placemark = ETree.Element("Placemark") @@ -133,7 +135,7 @@ def getItem(self, p): return ETree.tostring(placemark, encoding='unicode').replace('&', '&') # Helper method for getting additional fields in