Merge pull request #240 from ASFHyP3/develop

Release v0.14.0

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [0.14.0]
+### Added
+* `utils.get_esa_credentials` to check for the existence of CDSE credentials before processing begins.
+
+### Changed
+* Updated `hyp3lib` to  v2.0.2+, which uses the new Copernicus Data Space Ecosystem (CDSE) API to download orbit files.
+* Calls to `downloadSentinelOrbitFile` now specify the `esa_credentials` argument.
+
 ## [0.13.0]
 
 ### Changed

environment.yml

@@ -24,7 +24,7 @@ dependencies:
   # For running
   - gdal>=3
   - h5netcdf
-  - hyp3lib=1.7.0
+  - hyp3lib>=2,<3
   - isce2=2.6.1.dev7
   - autorift=1.5.0
   - opencv

pyproject.toml

@@ -26,14 +26,14 @@ dependencies = [
     'botocore',
     'gdal',
     'h5netcdf',
-    'hyp3lib==1.7.0',
     'matplotlib',
     'netCDF4',
     'numpy',
     'pyproj',
     'requests',
     'scipy',
     'xarray',
+    'hyp3lib>=2,<3',
 ]
 dynamic = ["version"]
 

src/hyp3_autorift/process.py

@@ -27,6 +27,7 @@
 
 from hyp3_autorift import geometry, image, io
 from hyp3_autorift.crop import crop_netcdf_product
+from hyp3_autorift.utils import get_esa_credentials
 
 log = logging.getLogger(__name__)
 
@@ -318,8 +319,14 @@ def apply_landsat_filtering(reference_path: str, secondary_path: str) \
     return reference_path, reference_zero_path, secondary_path, secondary_zero_path
 
 
-def process(reference: str, secondary: str, parameter_file: str = DEFAULT_PARAMETER_FILE,
-            naming_scheme: str = 'ITS_LIVE_OD') -> Tuple[Path, Path]:
+def process(
+    reference: str,
+    secondary: str,
+    parameter_file: str = DEFAULT_PARAMETER_FILE,
+    naming_scheme: str = 'ITS_LIVE_OD',
+    esa_username: Optional[str] = None,
+    esa_password: Optional[str] = None,
+) -> Tuple[Path, Path]:
     """Process a Sentinel-1, Sentinel-2, or Landsat-8 image pair
 
     Args:
@@ -348,9 +355,17 @@ def process(reference: str, secondary: str, parameter_file: str = DEFAULT_PARAME
 
         orbits = Path('Orbits').resolve()
         orbits.mkdir(parents=True, exist_ok=True)
-        reference_state_vec, reference_provider = downloadSentinelOrbitFile(reference, directory=str(orbits))
+
+        if (esa_username is None) or (esa_password is None):
+            esa_username, esa_password = get_esa_credentials()
+
+        reference_state_vec, reference_provider = downloadSentinelOrbitFile(
+            reference, directory=str(orbits), esa_credentials=(esa_username, esa_password)
+        )
         log.info(f'Downloaded orbit file {reference_state_vec} from {reference_provider}')
-        secondary_state_vec, secondary_provider = downloadSentinelOrbitFile(secondary, directory=str(orbits))
+        secondary_state_vec, secondary_provider = downloadSentinelOrbitFile(
+            secondary, directory=str(orbits), esa_credentials=(esa_username, esa_password)
+        )
         log.info(f'Downloaded orbit file {secondary_state_vec} from {secondary_provider}')
 
         polarization = get_s1_primary_polarization(reference)
@@ -505,6 +520,8 @@ def main():
     )
     parser.add_argument('--bucket', help='AWS bucket to upload product files to')
     parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files')
+    parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem")
+    parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem")
     parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE,
                         help='Shapefile for determining the correct search parameters by geographic location. '
                              'Path to shapefile must be understood by GDAL')

src/hyp3_autorift/s1_correction.py

@@ -3,6 +3,7 @@
 import logging
 from datetime import timedelta
 from pathlib import Path
+from typing import Optional
 
 from hyp3lib.aws import upload_file_to_s3
 from hyp3lib.fetch import download_file
@@ -11,19 +12,32 @@
 
 from hyp3_autorift import geometry, io
 from hyp3_autorift.process import DEFAULT_PARAMETER_FILE, get_s1_primary_polarization
+from hyp3_autorift.utils import get_esa_credentials
 from hyp3_autorift.vend.testGeogrid_ISCE import loadParsedata, runGeogrid
 log = logging.getLogger(__name__)
 
 
-def generate_correction_data(scene: str, buffer: int = 0, parameter_file: str = DEFAULT_PARAMETER_FILE):
+def generate_correction_data(
+    scene: str,
+    buffer: int = 0,
+    parameter_file: str = DEFAULT_PARAMETER_FILE,
+    esa_username: Optional[str] = None,
+    esa_password: Optional[str] = None,
+):
     scene_path = Path(f'{scene}.zip')
     if not scene_path.exists():
         scene_url = get_download_url(scene)
         scene_path = download_file(scene_url, chunk_size=5242880)
 
     orbits = Path('Orbits').resolve()
     orbits.mkdir(parents=True, exist_ok=True)
-    state_vec, oribit_provider = downloadSentinelOrbitFile(scene, directory=str(orbits))
+
+    if (esa_username is None) or (esa_password is None):
+        esa_username, esa_password = get_esa_credentials()
+
+    state_vec, oribit_provider = downloadSentinelOrbitFile(
+        scene, directory=str(orbits), esa_credentials=(esa_username, esa_password)
+    )
     log.info(f'Downloaded orbit file {state_vec} from {oribit_provider}')
 
     polarization = get_s1_primary_polarization(scene)
@@ -53,6 +67,8 @@ def main():
     )
     parser.add_argument('--bucket', help='AWS bucket to upload product files to')
     parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files')
+    parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem")
+    parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem")
     parser.add_argument('--buffer', type=int, default=0, help='Number of pixels to buffer each edge of the input scene')
     parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE,
                         help='Shapefile for determining the correct search parameters by geographic location. '

src/hyp3_autorift/utils.py

@@ -0,0 +1,30 @@
+import netrc
+import os
+from pathlib import Path
+from platform import system
+from typing import Tuple
+
+
+ESA_HOST = 'dataspace.copernicus.eu'
+
+
+def get_esa_credentials() -> Tuple[str, str]:
+    netrc_name = '_netrc' if system().lower() == 'windows' else '.netrc'
+    netrc_file = Path.home() / netrc_name
+
+    if "ESA_USERNAME" in os.environ and "ESA_PASSWORD" in os.environ:
+        username = os.environ["ESA_USERNAME"]
+        password = os.environ["ESA_PASSWORD"]
+        return username, password
+
+    if netrc_file.exists():
+        netrc_credentials = netrc.netrc(netrc_file)
+        if ESA_HOST in netrc_credentials.hosts:
+            username = netrc_credentials.hosts[ESA_HOST][0]
+            password = netrc_credentials.hosts[ESA_HOST][2]
+            return username, password
+
+    raise ValueError(
+        "Please provide Copernicus Data Space Ecosystem (CDSE) credentials via the "
+        "ESA_USERNAME and ESA_PASSWORD environment variables, or your netrc file."
+    )

tests/test_utils.py

@@ -0,0 +1,47 @@
+import pytest
+
+from hyp3_autorift.utils import ESA_HOST, get_esa_credentials
+
+
+def test_get_esa_credentials_env(tmp_path, monkeypatch):
+    with monkeypatch.context() as m:
+        m.setenv('ESA_USERNAME', 'foo')
+        m.setenv('ESA_PASSWORD', 'bar')
+        m.setenv('HOME', str(tmp_path))
+        (tmp_path / '.netrc').write_text(f'machine {ESA_HOST} login netrc_username password netrc_password')
+
+        username, password = get_esa_credentials()
+        assert username == 'foo'
+        assert password == 'bar'
+
+
+def test_get_esa_credentials_netrc(tmp_path, monkeypatch):
+    with monkeypatch.context() as m:
+        m.delenv('ESA_USERNAME', raising=False)
+        m.delenv('ESA_PASSWORD', raising=False)
+        m.setenv('HOME', str(tmp_path))
+        (tmp_path / '.netrc').write_text(f'machine {ESA_HOST} login foo password bar')
+
+        username, password = get_esa_credentials()
+        assert username == 'foo'
+        assert password == 'bar'
+
+
+def test_get_esa_credentials_missing(tmp_path, monkeypatch):
+    with monkeypatch.context() as m:
+        m.delenv('ESA_USERNAME', raising=False)
+        m.setenv('ESA_PASSWORD', 'env_password')
+        m.setenv('HOME', str(tmp_path))
+        (tmp_path / '.netrc').write_text('')
+        msg = 'Please provide.*'
+        with pytest.raises(ValueError, match=msg):
+            get_esa_credentials()
+
+    with monkeypatch.context() as m:
+        m.setenv('ESA_USERNAME', 'env_username')
+        m.delenv('ESA_PASSWORD', raising=False)
+        m.setenv('HOME', str(tmp_path))
+        (tmp_path / '.netrc').write_text('')
+        msg = 'Please provide.*'
+        with pytest.raises(ValueError, match=msg):
+            get_esa_credentials()