From a92d2b7dca3b06bd5ed083789bb49fc8c7b5f95d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:24:12 +0000 Subject: [PATCH 01/11] Bump ASFHyP3/actions from 0.11.2 to 0.12.0 Bumps [ASFHyP3/actions](https://github.com/asfhyp3/actions) from 0.11.2 to 0.12.0. - [Release notes](https://github.com/asfhyp3/actions/releases) - [Changelog](https://github.com/ASFHyP3/actions/blob/develop/CHANGELOG.md) - [Commits](https://github.com/asfhyp3/actions/compare/v0.11.2...v0.12.0) --- updated-dependencies: - dependency-name: ASFHyP3/actions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/changelog.yml | 2 +- .github/workflows/create-jira-issue.yml | 2 +- .github/workflows/labeled-pr.yml | 2 +- .github/workflows/release-template-comment.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/static-analysis.yml | 4 ++-- .github/workflows/tag-version.yml | 2 +- .github/workflows/test-and-build.yml | 6 +++--- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 7740de9e..3b1e7406 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,4 +13,4 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.12.0 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 99489d50..d95ef849 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.12.0 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index f89f3e3b..f408f3b3 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -12,4 +12,4 @@ on: jobs: call-labeled-pr-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.12.0 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml index dd3650ce..1f508a4d 100644 --- a/.github/workflows/release-template-comment.yml +++ b/.github/workflows/release-template-comment.yml @@ -7,6 +7,6 @@ on: jobs: call-release-checklist-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.12.0 secrets: USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 784648ef..7a005714 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.12.0 with: release_prefix: HyP3 autoRIFT secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index c7f96d1e..b89b9510 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,10 +4,10 @@ on: push jobs: call-flake8-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.12.0 with: local_package_names: hyp3_autorift excludes: src/hyp3_autorift/vend call-secrets-analysis-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.12.0 diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index 3b94479b..117193b5 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -7,6 +7,6 @@ on: jobs: call-bump-version-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.12.0 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index f23e4baf..095878d3 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -12,20 +12,20 @@ on: jobs: call-pytest-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.12.0 with: local_package_name: hyp3_autorift python_versions: >- ["3.9"] call-version-info-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.12.0 with: python_version: '3.9' call-docker-ghcr-workflow: needs: call-version-info-workflow - uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.2 + uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.12.0 with: version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }} secrets: From 3105f1dbe4ef87c490581180f4755e31c909573d Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 16 Dec 2024 13:34:25 -0500 Subject: [PATCH 02/11] switch to ruff from flake8 --- .github/workflows/static-analysis.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index b89b9510..1d08910e 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -3,11 +3,10 @@ name: Static analysis on: push jobs: - call-flake8-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.12.0 - with: - local_package_names: hyp3_autorift - excludes: src/hyp3_autorift/vend - call-secrets-analysis-workflow: + # Docs: https://github.com/ASFHyP3/actions uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.12.0 + + call-ruff-workflow: + # Docs: https://github.com/ASFHyP3/actions + uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.12.0 From 6cd5f0a7751b4932a3638114385a8a55c73d2e04 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 16 Dec 2024 13:34:48 -0500 Subject: [PATCH 03/11] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cb3d2dc..812e0b49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.21.1] +### Changed +- The [`static-analysis`](.github/workflows/static-analysis.yml) Github Actions workflow now uses `ruff` rather than `flake8` for linting. + ## [0.21.0] ### Added * Logger is now configured in process.main() so paths to reference/secondary scenes will now be logged. From dbb6568c203c9cace6b53a9d10612d0abf90bcea Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 16 Dec 2024 13:35:20 -0500 Subject: [PATCH 04/11] add ruff config --- ruff.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 ruff.toml diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..61f91605 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,4 @@ +line-length = 120 + +[format] +quote-style = "single" From 7f10d09abbe55a12d4db66f518fa0014af54a7fb Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 16 Dec 2024 13:39:29 -0500 Subject: [PATCH 05/11] formatting for ruff --- src/hyp3_autorift/__init__.py | 12 ++- src/hyp3_autorift/__main__.py | 11 +- src/hyp3_autorift/crop.py | 12 +-- src/hyp3_autorift/geometry.py | 5 +- src/hyp3_autorift/image.py | 63 ++++++------ src/hyp3_autorift/process.py | 102 +++++++++++-------- src/hyp3_autorift/s1_correction.py | 13 +-- src/hyp3_autorift/s1_isce2.py | 141 +++++++++++++++++--------- src/hyp3_autorift/utils.py | 37 ++++--- tests/test_geometry.py | 8 +- tests/test_process.py | 156 ++++++++++++++++++----------- tests/test_s1_isce2.py | 24 +++-- 12 files changed, 350 insertions(+), 234 deletions(-) diff --git a/src/hyp3_autorift/__init__.py b/src/hyp3_autorift/__init__.py index aa94c161..c121b609 100644 --- a/src/hyp3_autorift/__init__.py +++ b/src/hyp3_autorift/__init__.py @@ -5,11 +5,13 @@ try: __version__ = version(__name__) except PackageNotFoundError: - print(f'{__name__} package is not installed!\n' - f'Install in editable/develop mode via (from the top of this repo):\n' - f' python -m pip install -e .[develop]\n' - f'Or, to just get the version number use:\n' - f' python setup.py --version') + print( + f'{__name__} package is not installed!\n' + f'Install in editable/develop mode via (from the top of this repo):\n' + f' python -m pip install -e .[develop]\n' + f'Or, to just get the version number use:\n' + f' python setup.py --version' + ) __all__ = [ '__version__', diff --git a/src/hyp3_autorift/__main__.py b/src/hyp3_autorift/__main__.py index b49e65bf..3405c06f 100644 --- a/src/hyp3_autorift/__main__.py +++ b/src/hyp3_autorift/__main__.py @@ -2,7 +2,6 @@ AutoRIFT processing for HyP3 """ - import argparse import os import sys @@ -16,8 +15,10 @@ def main(): parser = argparse.ArgumentParser(prefix_chars='+', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '++process', choices=['hyp3_autorift', 's1_correction'], default='hyp3_autorift', - help='Select the console_script entrypoint to use' # console_script entrypoints are specified in `setup.py` + '++process', + choices=['hyp3_autorift', 's1_correction'], + default='hyp3_autorift', + help='Select the console_script entrypoint to use', # console_script entrypoints are specified in `setup.py` ) parser.add_argument('++omp-num-threads', type=int, help='The number of OpenMP threads to use for parallel regions') @@ -38,9 +39,7 @@ def main(): (process_entry_point,) = {process for process in eps if process.name == args.process} sys.argv = [args.process, *unknowns] - sys.exit( - process_entry_point.load()() - ) + sys.exit(process_entry_point.load()()) if __name__ == '__main__': diff --git a/src/hyp3_autorift/crop.py b/src/hyp3_autorift/crop.py index 58a725dc..5e2e8273 100644 --- a/src/hyp3_autorift/crop.py +++ b/src/hyp3_autorift/crop.py @@ -36,7 +36,7 @@ import pyproj import xarray as xr -ENCODING_ATTRS = ['_FillValue', 'dtype', "zlib", "complevel", "shuffle", 'add_offset', 'scale_factor'] +ENCODING_ATTRS = ['_FillValue', 'dtype', 'zlib', 'complevel', 'shuffle', 'add_offset', 'scale_factor'] def crop_netcdf_product(netcdf_file: Path) -> Path: @@ -58,7 +58,7 @@ def crop_netcdf_product(netcdf_file: Path) -> Path: # Based on X/Y extends, mask original dataset mask_lon = (ds.x >= grid_x_min) & (ds.x <= grid_x_max) mask_lat = (ds.y >= grid_y_min) & (ds.y <= grid_y_max) - mask = (mask_lon & mask_lat) + mask = mask_lon & mask_lat cropped_ds = ds.where(mask).dropna(dim='x', how='all').dropna(dim='y', how='all') cropped_ds = cropped_ds.load() @@ -74,11 +74,7 @@ def crop_netcdf_product(netcdf_file: Path) -> Path: # Convert to lon/lat coordinates projection = ds['mapping'].attrs['spatial_epsg'] - to_lon_lat_transformer = pyproj.Transformer.from_crs( - f"EPSG:{projection}", - 'EPSG:4326', - always_xy=True - ) + to_lon_lat_transformer = pyproj.Transformer.from_crs(f'EPSG:{projection}', 'EPSG:4326', always_xy=True) # Update centroid information for the granule center_lon_lat = to_lon_lat_transformer.transform(center_x, center_y) @@ -91,7 +87,7 @@ def crop_netcdf_product(netcdf_file: Path) -> Path: y_cell = y_values[1] - y_values[0] # It was decided to keep all values in GeoTransform center-based - cropped_ds['mapping'].attrs['GeoTransform'] = f"{x_values[0]} {x_cell} 0 {y_values[0]} 0 {y_cell}" + cropped_ds['mapping'].attrs['GeoTransform'] = f'{x_values[0]} {x_cell} 0 {y_values[0]} 0 {y_cell}' # Compute chunking like AutoRIFT does: # https://github.com/ASFHyP3/hyp3-autorift/blob/develop/hyp3_autorift/vend/netcdf_output.py#L410-L411 diff --git a/src/hyp3_autorift/geometry.py b/src/hyp3_autorift/geometry.py index 27e29a0f..97846a94 100644 --- a/src/hyp3_autorift/geometry.py +++ b/src/hyp3_autorift/geometry.py @@ -9,8 +9,9 @@ log = logging.getLogger(__name__) -def polygon_from_bbox(x_limits: Tuple[float, float], y_limits: Tuple[float, float], - epsg_code: int = 4326) -> ogr.Geometry: +def polygon_from_bbox( + x_limits: Tuple[float, float], y_limits: Tuple[float, float], epsg_code: int = 4326 +) -> ogr.Geometry: ring = ogr.Geometry(ogr.wkbLinearRing) ring.AddPoint_2D(x_limits[0], y_limits[1]) ring.AddPoint_2D(x_limits[1], y_limits[1]) diff --git a/src/hyp3_autorift/image.py b/src/hyp3_autorift/image.py index a80c5985..eb901dac 100644 --- a/src/hyp3_autorift/image.py +++ b/src/hyp3_autorift/image.py @@ -6,38 +6,41 @@ from matplotlib.colors import LinearSegmentedColormap from scipy.interpolate import PchipInterpolator -COLOR_MAP = np.array([ - # data value, R, G, B, A - [0, 255, 255, 255, 0], - [2, 166, 238, 255, 255], - [4, 97, 195, 219, 255], - [9, 84, 169, 254, 255], - [16, 84, 130, 254, 255], - [25, 84, 85, 254, 255], - [36, 50, 119, 220, 255], - [49, 16, 153, 186, 255], - [64, 16, 186, 153, 255], - [81, 50, 220, 119, 255], - [100, 84, 254, 85, 255], - [121, 118, 221, 51, 255], - [144, 153, 186, 16, 255], - [169, 187, 152, 17, 255], - [196, 221, 118, 51, 255], - [225, 255, 85, 85, 255], - [289, 255, 25, 85, 255], - [324, 213, 1, 72, 255], - [361, 158, 1, 66, 255], - [400, 140, 0, 51, 255], - [441, 122, 0, 166, 255], - [484, 140, 0, 191, 255], - [529, 159, 0, 217, 255], - [576, 213, 0, 255, 255], - [625, 255, 0, 138, 255], -]) +COLOR_MAP = np.array( + [ + # data value, R, G, B, A + [0, 255, 255, 255, 0], + [2, 166, 238, 255, 255], + [4, 97, 195, 219, 255], + [9, 84, 169, 254, 255], + [16, 84, 130, 254, 255], + [25, 84, 85, 254, 255], + [36, 50, 119, 220, 255], + [49, 16, 153, 186, 255], + [64, 16, 186, 153, 255], + [81, 50, 220, 119, 255], + [100, 84, 254, 85, 255], + [121, 118, 221, 51, 255], + [144, 153, 186, 16, 255], + [169, 187, 152, 17, 255], + [196, 221, 118, 51, 255], + [225, 255, 85, 85, 255], + [289, 255, 25, 85, 255], + [324, 213, 1, 72, 255], + [361, 158, 1, 66, 255], + [400, 140, 0, 51, 255], + [441, 122, 0, 166, 255], + [484, 140, 0, 191, 255], + [529, 159, 0, 217, 255], + [576, 213, 0, 255, 255], + [625, 255, 0, 138, 255], + ] +) -def make_browse(out_file: Path, data: np.ndarray, - min_value: Optional[float] = None, max_value: Optional[float] = 625.) -> Path: +def make_browse( + out_file: Path, data: np.ndarray, min_value: Optional[float] = None, max_value: Optional[float] = 625.0 +) -> Path: data_values = COLOR_MAP[:, 0] pchip = PchipInterpolator(data_values, np.linspace(0, 1, len(data_values))) image = pchip(np.clip(data, min_value, max_value)) diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 840efb9f..6e1fd3ab 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -40,8 +40,9 @@ 'L4': {'T': 'tm', 'M': 'mss'}, } -DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.s3.amazonaws.com/' \ - 'autorift_parameters/v001/autorift_landice_0120m.shp' +DEFAULT_PARAMETER_FILE = ( + '/vsicurl/http://its-live-data.s3.amazonaws.com/' 'autorift_parameters/v001/autorift_landice_0120m.shp' +) PLATFORM_SHORTNAME_LONGNAME_MAPPING = { 'S1': 'sentinel1', @@ -117,8 +118,11 @@ def get_s2_path(scene_name: str) -> str: manifest_text = get_s2_manifest(scene_name) root = ET.fromstring(manifest_text) elements = root.findall(".//fileLocation[@locatorType='URL'][@href]") - hrefs = [element.attrib['href'] for element in elements if - element.attrib['href'].endswith('_B08.jp2') and '/IMG_DATA/' in element.attrib['href']] + hrefs = [ + element.attrib['href'] + for element in elements + if element.attrib['href'].endswith('_B08.jp2') and '/IMG_DATA/' in element.attrib['href'] + ] if len(hrefs) == 1: # post-2016-12-06 scene; only one tile file_path = hrefs[0] @@ -253,20 +257,21 @@ def _apply_filter_function(image_path: str, filter_function: Callable) -> Tuple[ image_filtered, zero_mask = filter_function(image_array, image_nodata) image_new_path = create_filtered_filepath(image_path) - _ = utils.write_geospatial(image_new_path, image_filtered, image_transform, image_projection, - nodata=None, dtype=gdal.GDT_Float32) + _ = utils.write_geospatial( + image_new_path, image_filtered, image_transform, image_projection, nodata=None, dtype=gdal.GDT_Float32 + ) zero_path = None if zero_mask is not None: zero_path = create_filtered_filepath(f'{Path(image_new_path).stem}_zeroMask{Path(image_new_path).suffix}') - _ = utils.write_geospatial(zero_path, zero_mask, image_transform, image_projection, - nodata=np.iinfo(np.uint8).max, dtype=gdal.GDT_Byte) + _ = utils.write_geospatial( + zero_path, zero_mask, image_transform, image_projection, nodata=np.iinfo(np.uint8).max, dtype=gdal.GDT_Byte + ) return image_new_path, zero_path -def apply_landsat_filtering(reference_path: str, secondary_path: str) \ - -> Tuple[str, Optional[str], str, Optional[str]]: +def apply_landsat_filtering(reference_path: str, secondary_path: str) -> Tuple[str, Optional[str], str, Optional[str]]: reference_platform = get_platform(Path(reference_path).name) secondary_platform = get_platform(Path(secondary_path).name) if reference_platform > 'L7' and secondary_platform > 'L7': @@ -309,11 +314,11 @@ def point_to_region(lat: float, lon: float) -> str: nw_hemisphere = 'N' if lat >= 0.0 else 'S' ew_hemisphere = 'E' if lon >= 0.0 else 'W' - region_lat = int(10*np.trunc(np.abs(lat/10.0))) + region_lat = int(10 * np.trunc(np.abs(lat / 10.0))) if region_lat == 90: # if you are exactly at a pole, put in lat = 80 bin region_lat = 80 - region_lon = int(10*np.trunc(np.abs(lon/10.0))) + region_lon = int(10 * np.trunc(np.abs(lon / 10.0))) if region_lon >= 180: # if you are at the dateline, back off to the 170 bin region_lon = 170 @@ -329,12 +334,7 @@ def get_opendata_prefix(file: Path): lat, lon = get_lat_lon_from_ncfile(file) region = point_to_region(lat, lon) - return '/'.join([ - 'velocity_image_pair', - PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname], - 'v02', - region - ]) + return '/'.join(['velocity_image_pair', PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname], 'v02', region]) def process( @@ -365,6 +365,7 @@ def process( if platform == 'S1': from hyp3_autorift.s1_isce2 import process_sentinel1_with_isce2 + netcdf_file = process_sentinel1_with_isce2(reference, secondary, parameter_file) else: @@ -398,8 +399,9 @@ def process( # Log path here before we transform it log.info(f'Reference scene path: {reference_path}') log.info(f'Secondary scene path: {secondary_path}') - reference_path, reference_zero_path, secondary_path, secondary_zero_path = \ - apply_landsat_filtering(reference_path, secondary_path) + reference_path, reference_zero_path, secondary_path, secondary_zero_path = apply_landsat_filtering( + reference_path, secondary_path + ) if reference_metadata['properties']['proj:epsg'] != secondary_metadata['properties']['proj:epsg']: log.info('Reference and secondary projections are different! Reprojecting.') @@ -420,20 +422,28 @@ def process( scene_poly = geometry.polygon_from_bbox(x_limits=lat_limits, y_limits=lon_limits) parameter_info = utils.find_jpl_parameter_info(scene_poly, parameter_file) - from hyp3_autorift.vend.testGeogridOptical import ( - coregisterLoadMetadata, runGeogrid) + from hyp3_autorift.vend.testGeogridOptical import coregisterLoadMetadata, runGeogrid + meta_r, meta_s = coregisterLoadMetadata( - reference_path, secondary_path, + reference_path, + secondary_path, reference_metadata=reference_metadata, secondary_metadata=secondary_metadata, ) geogrid_info = runGeogrid(meta_r, meta_s, epsg=parameter_info['epsg'], **parameter_info['geogrid']) from hyp3_autorift.vend.testautoRIFT import generateAutoriftProduct + netcdf_file = generateAutoriftProduct( - reference_path, secondary_path, nc_sensor=platform, optical_flag=True, ncname=None, - reference_metadata=reference_metadata, secondary_metadata=secondary_metadata, - geogrid_run_info=geogrid_info, **parameter_info['autorift'], + reference_path, + secondary_path, + nc_sensor=platform, + optical_flag=True, + ncname=None, + reference_metadata=reference_metadata, + secondary_metadata=secondary_metadata, + geogrid_run_info=geogrid_info, + **parameter_info['autorift'], parameter_file=DEFAULT_PARAMETER_FILE.replace('/vsicurl/', ''), ) @@ -458,7 +468,7 @@ def process( with Dataset(product_file) as nc: velocity = nc.variables['v'] - data = np.ma.masked_values(velocity, -32767.).filled(0) + data = np.ma.masked_values(velocity, -32767.0).filled(0) browse_file = product_file.with_suffix('.png') image.make_browse(browse_file, data) @@ -469,29 +479,37 @@ def process( def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') - parser.add_argument('--publish-bucket', default='', - help='Additionally, publish products to this bucket. Necessary credentials must be provided ' - 'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.') - parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, - help='Shapefile for determining the correct search parameters by geographic location. ' - 'Path to shapefile must be understood by GDAL') - parser.add_argument('--naming-scheme', default='ITS_LIVE_OD', choices=['ITS_LIVE_OD', 'ITS_LIVE_PROD'], - help='Naming scheme to use for product files') - parser.add_argument('granules', type=str.split, nargs='+', - help='Granule pair to process') + parser.add_argument( + '--publish-bucket', + default='', + help='Additionally, publish products to this bucket. Necessary credentials must be provided ' + 'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.', + ) + parser.add_argument( + '--parameter-file', + default=DEFAULT_PARAMETER_FILE, + help='Shapefile for determining the correct search parameters by geographic location. ' + 'Path to shapefile must be understood by GDAL', + ) + parser.add_argument( + '--naming-scheme', + default='ITS_LIVE_OD', + choices=['ITS_LIVE_OD', 'ITS_LIVE_PROD'], + help='Naming scheme to use for product files', + ) + parser.add_argument('granules', type=str.split, nargs='+', help='Granule pair to process') args = parser.parse_args() args.granules = [item for sublist in args.granules for item in sublist] if len(args.granules) != 2: parser.error('Must provide exactly two granules') - logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', - datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO) + logging.basicConfig( + format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO + ) g1, g2 = sorted(args.granules, key=get_datetime) diff --git a/src/hyp3_autorift/s1_correction.py b/src/hyp3_autorift/s1_correction.py index fd785566..cc6f0470 100644 --- a/src/hyp3_autorift/s1_correction.py +++ b/src/hyp3_autorift/s1_correction.py @@ -11,15 +11,16 @@ def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--bucket', help='AWS bucket to upload product files to') parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files') parser.add_argument('--buffer', type=int, default=0, help='Number of pixels to buffer each edge of the input scene') - parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE, - help='Shapefile for determining the correct search parameters by geographic location. ' - 'Path to shapefile must be understood by GDAL') + parser.add_argument( + '--parameter-file', + default=DEFAULT_PARAMETER_FILE, + help='Shapefile for determining the correct search parameters by geographic location. ' + 'Path to shapefile must be understood by GDAL', + ) parser.add_argument('granule', help='Reference granule to process') args = parser.parse_args() diff --git a/src/hyp3_autorift/s1_isce2.py b/src/hyp3_autorift/s1_isce2.py index 7c340781..3daa8e78 100644 --- a/src/hyp3_autorift/s1_isce2.py +++ b/src/hyp3_autorift/s1_isce2.py @@ -77,35 +77,39 @@ def process_sentinel1_with_isce2(reference, secondary, parameter_file): gdal.AllRegister() netcdf_file = generateAutoriftProduct( - reference_path, secondary_path, nc_sensor='S1', optical_flag=False, ncname=None, - geogrid_run_info=geogrid_info, **parameter_info['autorift'], - parameter_file=parameter_file.replace('/vsicurl/', ''), - ) + reference_path, + secondary_path, + nc_sensor='S1', + optical_flag=False, + ncname=None, + geogrid_run_info=geogrid_info, + **parameter_info['autorift'], + parameter_file=parameter_file.replace('/vsicurl/', ''), + ) return netcdf_file def write_conversion_file( - *, - file_name: str, - srs: osr.SpatialReference, - epsg: int, - tran: List[float], - x: np.ndarray, - y: np.ndarray, - M11: np.ndarray, - M12: np.ndarray, - dr_2_vr_factor: float, - ChunkSize: List[int], - NoDataValue: int = -32767, - noDataMask: np.ndarray, - parameter_file: str + *, + file_name: str, + srs: osr.SpatialReference, + epsg: int, + tran: List[float], + x: np.ndarray, + y: np.ndarray, + M11: np.ndarray, + M12: np.ndarray, + dr_2_vr_factor: float, + ChunkSize: List[int], + NoDataValue: int = -32767, + noDataMask: np.ndarray, + parameter_file: str, ) -> str: - nc_outfile = Dataset(file_name, 'w', clobber=True, format='NETCDF4') nc_outfile.setncattr('GDAL_AREA_OR_POINT', 'Area') nc_outfile.setncattr('Conventions', 'CF-1.8') - nc_outfile.setncattr('date_created', datetime.now().strftime("%d-%b-%Y %H:%M:%S")) + nc_outfile.setncattr('date_created', datetime.now().strftime('%d-%b-%Y %H:%M:%S')) nc_outfile.setncattr('title', 'autoRIFT S1 Corrections') nc_outfile.setncattr('autoRIFT_software_version', version) nc_outfile.setncattr('autoRIFT_parameter_file', parameter_file) @@ -162,36 +166,56 @@ def write_conversion_file( else: raise Exception(f'Projection {srs.GetAttrValue("PROJECTION")} not recognized for this program') - var = nc_outfile.createVariable('M11', np.dtype('float32'), ('y', 'x'), fill_value=NoDataValue, - zlib=True, complevel=2, shuffle=True, chunksizes=ChunkSize) + var = nc_outfile.createVariable( + 'M11', + np.dtype('float32'), + ('y', 'x'), + fill_value=NoDataValue, + zlib=True, + complevel=2, + shuffle=True, + chunksizes=ChunkSize, + ) var.setncattr('standard_name', 'conversion_matrix_element_11') var.setncattr( 'description', 'conversion matrix element (1st row, 1st column) that can be multiplied with vx to give range pixel ' - 'displacement dr (see Eq. A18 in https://www.mdpi.com/2072-4292/13/4/749)' + 'displacement dr (see Eq. A18 in https://www.mdpi.com/2072-4292/13/4/749)', ) var.setncattr('units', 'pixel/(meter/year)') var.setncattr('grid_mapping', mapping_var_name) var.setncattr('dr_to_vr_factor', dr_2_vr_factor) - var.setncattr('dr_to_vr_factor_description', 'multiplicative factor that converts slant range ' - 'pixel displacement dr to slant range velocity vr') + var.setncattr( + 'dr_to_vr_factor_description', + 'multiplicative factor that converts slant range ' 'pixel displacement dr to slant range velocity vr', + ) M11[noDataMask] = NoDataValue var[:] = M11 - var = nc_outfile.createVariable('M12', np.dtype('float32'), ('y', 'x'), fill_value=NoDataValue, - zlib=True, complevel=2, shuffle=True, chunksizes=ChunkSize) + var = nc_outfile.createVariable( + 'M12', + np.dtype('float32'), + ('y', 'x'), + fill_value=NoDataValue, + zlib=True, + complevel=2, + shuffle=True, + chunksizes=ChunkSize, + ) var.setncattr('standard_name', 'conversion_matrix_element_12') var.setncattr( 'description', 'conversion matrix element (1st row, 2nd column) that can be multiplied with vy to give range pixel ' - 'displacement dr (see Eq. A18 in https://www.mdpi.com/2072-4292/13/4/749)' + 'displacement dr (see Eq. A18 in https://www.mdpi.com/2072-4292/13/4/749)', ) var.setncattr('units', 'pixel/(meter/year)') var.setncattr('grid_mapping', mapping_var_name) var.setncattr('dr_to_vr_factor', dr_2_vr_factor) - var.setncattr('dr_to_vr_factor_description', - 'multiplicative factor that converts slant range pixel displacement dr to slant range velocity vr') + var.setncattr( + 'dr_to_vr_factor_description', + 'multiplicative factor that converts slant range pixel displacement dr to slant range velocity vr', + ) M12[noDataMask] = NoDataValue var[:] = M12 @@ -203,15 +227,15 @@ def write_conversion_file( def create_conversion_matrices( - *, - scene: str, - grid_location: str = 'window_location.tif', - offset2vx: str = 'window_rdr_off2vel_x_vec.tif', - offset2vy: str = 'window_rdr_off2vel_y_vec.tif', - scale_factor: str = 'window_scale_factor.tif', - epsg: int = 4326, - parameter_file: str = DEFAULT_PARAMETER_FILE, - **kwargs, + *, + scene: str, + grid_location: str = 'window_location.tif', + offset2vx: str = 'window_rdr_off2vel_x_vec.tif', + offset2vy: str = 'window_rdr_off2vel_y_vec.tif', + scale_factor: str = 'window_scale_factor.tif', + epsg: int = 4326, + parameter_file: str = DEFAULT_PARAMETER_FILE, + **kwargs, ) -> Path: xGrid, tran, _, srs, nodata = utils.load_geospatial(grid_location, band=1) @@ -251,8 +275,18 @@ def create_conversion_matrices( dr_2_vr_factor = np.median(offset2vr[np.logical_not(np.isnan(offset2vr))]) conversion_nc = write_conversion_file( - file_name='conversion_matrices.nc', srs=srs, epsg=epsg, tran=tran, x=x, y=y, M11=M11, M12=M12, - dr_2_vr_factor=dr_2_vr_factor, ChunkSize=ChunkSize, noDataMask=noDataMask, parameter_file=parameter_file, + file_name='conversion_matrices.nc', + srs=srs, + epsg=epsg, + tran=tran, + x=x, + y=y, + M11=M11, + M12=M12, + dr_2_vr_factor=dr_2_vr_factor, + ChunkSize=ChunkSize, + noDataMask=noDataMask, + parameter_file=parameter_file, ) return Path(conversion_nc) @@ -264,6 +298,7 @@ def generate_correction_data( parameter_file: str = DEFAULT_PARAMETER_FILE, ) -> (dict, Path): from hyp3_autorift.vend.testGeogrid_ISCE import loadParsedata, runGeogrid + scene_path = Path(f'{scene}.zip') if not scene_path.exists(): scene_url = get_download_url(scene) @@ -310,6 +345,7 @@ class SysArgvManager: A bug in the ISCE2 Application class causes sys.argv to always be parsed when no options are proved, even when setting `cmdline=[]`, preventing programmatic use. """ + def __init__(self): self.argv = sys.argv.copy() @@ -322,8 +358,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): def get_topsinsar_config(): from isce.applications.topsApp import TopsInSAR + with SysArgvManager(): - insar = TopsInSAR(name="topsApp") + insar = TopsInSAR(name='topsApp') insar.configure() config_data = {} @@ -335,9 +372,7 @@ def get_topsinsar_config(): scene.configure() scene.swathNumber = swath scene.parse() - sensing_times.append( - (scene.product.sensingStart, scene.product.sensingStop) - ) + sensing_times.append((scene.product.sensingStart, scene.product.sensingStop)) sensing_start = min([sensing_time[0] for sensing_time in sensing_times]) sensing_stop = max([sensing_time[1] for sensing_time in sensing_times]) @@ -345,7 +380,7 @@ def get_topsinsar_config(): sensing_dt = (sensing_stop - sensing_start) / 2 + sensing_start config_data[f'{name}_filename'] = Path(scene.safe[0]).name - config_data[f'{name}_dt'] = sensing_dt.strftime("%Y%m%dT%H:%M:%S.%f").rstrip('0') + config_data[f'{name}_dt'] = sensing_dt.strftime('%Y%m%dT%H:%M:%S.%f').rstrip('0') return config_data @@ -405,6 +440,7 @@ def bounding_box(safe, priority='reference', polarization='hh', orbits='Orbits', from contrib.geo_autoRIFT.geogrid import Geogrid from isceobj.Orbit.Orbit import Orbit from isceobj.Sensor.TOPS.Sentinel1 import Sentinel1 + frames = [] for swath in range(1, 4): rdr = Sentinel1() @@ -447,7 +483,7 @@ def bounding_box(safe, priority='reference', polarization='hh', orbits='Orbits', obj.prf = prf obj.lookSide = -1 obj.numberOfLines = int(np.round((sensing_stop - sensing_start).total_seconds() * prf)) - obj.numberOfSamples = int(np.round((far_range - starting_range)/range_pixel_size)) + obj.numberOfSamples = int(np.round((far_range - starting_range) / range_pixel_size)) obj.orbit = orb obj.epsg = epsg @@ -476,9 +512,14 @@ def prep_isce_dem(input_dem, lat_limits, lon_limits, isce_dem=None): in_ds = gdal.OpenShared(input_dem, gdal.GA_ReadOnly) warp_options = gdal.WarpOptions( - format='ENVI', outputType=gdal.GDT_Int16, resampleAlg='cubic', - xRes=0.001, yRes=0.001, dstSRS='EPSG:4326', dstNodata=0, - outputBounds=[lon_limits[0], lat_limits[0], lon_limits[1], lat_limits[1]] + format='ENVI', + outputType=gdal.GDT_Int16, + resampleAlg='cubic', + xRes=0.001, + yRes=0.001, + dstSRS='EPSG:4326', + dstNodata=0, + outputBounds=[lon_limits[0], lat_limits[0], lon_limits[1], lat_limits[1]], ) gdal.Warp(isce_dem, in_ds, options=warp_options) diff --git a/src/hyp3_autorift/utils.py b/src/hyp3_autorift/utils.py index a8d04a5c..f7eeb4fa 100644 --- a/src/hyp3_autorift/utils.py +++ b/src/hyp3_autorift/utils.py @@ -78,14 +78,14 @@ def find_jpl_parameter_info(polygon: ogr.Geometry, parameter_file: str) -> dict: 'stable_surface_mask': 'window_stable_surface_mask.tif', 'scale_factor': 'window_scale_factor.tif', 'mpflag': 0, - } + }, } break if parameter_info is None: - raise DemError('Could not determine appropriate DEM for:\n' - f' centroid: {centroid}' - f' using: {parameter_file}') + raise DemError( + 'Could not determine appropriate DEM for:\n' f' centroid: {centroid}' f' using: {parameter_file}' + ) dem_geotransform = gdal.Info(parameter_info['geogrid']['dem'], format='json')['geoTransform'] parameter_info['xsize'] = abs(dem_geotransform[1]) @@ -108,8 +108,9 @@ def load_geospatial(infile: str, band: int = 1): return data, transform, projection, srs, nodata -def write_geospatial(outfile: str, data, transform, projection, nodata, - driver: str = 'GTiff', dtype: int = gdal.GDT_Float64) -> str: +def write_geospatial( + outfile: str, data, transform, projection, nodata, driver: str = 'GTiff', dtype: int = gdal.GDT_Float64 +) -> str: driver = gdal.GetDriverByName(driver) rows, cols = data.shape @@ -146,11 +147,23 @@ def ensure_same_projection(reference_path: Union[str, Path], secondary_path: Uni reprojected_reference = str(reprojection_dir / Path(reference_path).name) reprojected_secondary = str(reprojection_dir / Path(secondary_path).name) - gdal.Warp(reprojected_reference, str(reference_path), dstSRS=f'EPSG:{ref_epsg}', - xRes=ref_info['geoTransform'][1], yRes=ref_info['geoTransform'][5], - resampleAlg='lanczos', targetAlignedPixels=True) - gdal.Warp(reprojected_secondary, str(secondary_path), dstSRS=f'EPSG:{ref_epsg}', - xRes=ref_info['geoTransform'][1], yRes=ref_info['geoTransform'][5], - resampleAlg='lanczos', targetAlignedPixels=True) + gdal.Warp( + reprojected_reference, + str(reference_path), + dstSRS=f'EPSG:{ref_epsg}', + xRes=ref_info['geoTransform'][1], + yRes=ref_info['geoTransform'][5], + resampleAlg='lanczos', + targetAlignedPixels=True, + ) + gdal.Warp( + reprojected_secondary, + str(secondary_path), + dstSRS=f'EPSG:{ref_epsg}', + xRes=ref_info['geoTransform'][1], + yRes=ref_info['geoTransform'][5], + resampleAlg='lanczos', + targetAlignedPixels=True, + ) return reprojected_reference, reprojected_secondary diff --git a/tests/test_geometry.py b/tests/test_geometry.py index f114e585..22a02044 100644 --- a/tests/test_geometry.py +++ b/tests/test_geometry.py @@ -27,7 +27,7 @@ def test_pol_bounds_in_proj(): polygon = geometry.polygon_from_bbox(x_limits=lat_limits, y_limits=lon_limits) assert np.allclose( geometry.poly_bounds_in_proj(polygon, out_epsg=3413), # NPS - (3776365.5697414433, 3899644.3388010086, -340706.3423259673, -264432.19003121805) + (3776365.5697414433, 3899644.3388010086, -340706.3423259673, -264432.19003121805), ) lat_limits = (-58, -57) @@ -35,7 +35,7 @@ def test_pol_bounds_in_proj(): polygon = geometry.polygon_from_bbox(x_limits=lat_limits, y_limits=lon_limits) assert np.allclose( geometry.poly_bounds_in_proj(polygon, out_epsg=3031), # SPS - (2292512.6214329866, 2416952.768825992, 2691684.1770189586, 2822144.2827928355) + (2292512.6214329866, 2416952.768825992, 2691684.1770189586, 2822144.2827928355), ) lat_limits = (22, 23) @@ -43,7 +43,7 @@ def test_pol_bounds_in_proj(): polygon = geometry.polygon_from_bbox(x_limits=lat_limits, y_limits=lon_limits) assert np.allclose( geometry.poly_bounds_in_proj(polygon, out_epsg=32637), - (602485.1663686256, 706472.0593133729, 2433164.428653589, 2544918.1043369616) + (602485.1663686256, 706472.0593133729, 2433164.428653589, 2544918.1043369616), ) lat_limits = (-23, -22) @@ -51,7 +51,7 @@ def test_pol_bounds_in_proj(): polygon = geometry.polygon_from_bbox(x_limits=lat_limits, y_limits=lon_limits) assert np.allclose( geometry.poly_bounds_in_proj(polygon, out_epsg=32737), - (602485.1663686256, 706472.0593133729, 7455081.895663038, 7566835.5713464115) + (602485.1663686256, 706472.0593133729, 7455081.895663038, 7566835.5713464115), ) diff --git a/tests/test_process.py b/tests/test_process.py index 9d5d5e52..5b607d55 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -38,56 +38,80 @@ def test_get_platform(): def test_get_lc2_stac_json_key(): - expected = 'collection02/level-1/standard/oli-tirs/2021/122/028/LC09_L1GT_122028_20211107_20220119_02_T2/' \ - 'LC09_L1GT_122028_20211107_20220119_02_T2_stac.json' + expected = ( + 'collection02/level-1/standard/oli-tirs/2021/122/028/LC09_L1GT_122028_20211107_20220119_02_T2/' + 'LC09_L1GT_122028_20211107_20220119_02_T2_stac.json' + ) assert process.get_lc2_stac_json_key('LC09_L1GT_122028_20211107_20220119_02_T2') == expected - expected = 'collection02/level-1/standard/oli-tirs/2022/060/002/LO09_L1TP_060002_20220316_20220316_02_T1/' \ - 'LO09_L1TP_060002_20220316_20220316_02_T1_stac.json' + expected = ( + 'collection02/level-1/standard/oli-tirs/2022/060/002/LO09_L1TP_060002_20220316_20220316_02_T1/' + 'LO09_L1TP_060002_20220316_20220316_02_T1_stac.json' + ) assert process.get_lc2_stac_json_key('LO09_L1TP_060002_20220316_20220316_02_T1') == expected - expected = 'collection02/level-1/standard/oli-tirs/2022/137/206/LT09_L1GT_137206_20220107_20220123_02_T2/' \ - 'LT09_L1GT_137206_20220107_20220123_02_T2_stac.json' + expected = ( + 'collection02/level-1/standard/oli-tirs/2022/137/206/LT09_L1GT_137206_20220107_20220123_02_T2/' + 'LT09_L1GT_137206_20220107_20220123_02_T2_stac.json' + ) assert process.get_lc2_stac_json_key('LT09_L1GT_137206_20220107_20220123_02_T2') == expected - expected = 'collection02/level-1/standard/oli-tirs/2016/138/039/LC08_L1TP_138039_20161105_20200905_02_T1/' \ - 'LC08_L1TP_138039_20161105_20200905_02_T1_stac.json' + expected = ( + 'collection02/level-1/standard/oli-tirs/2016/138/039/LC08_L1TP_138039_20161105_20200905_02_T1/' + 'LC08_L1TP_138039_20161105_20200905_02_T1_stac.json' + ) assert process.get_lc2_stac_json_key('LC08_L1TP_138039_20161105_20200905_02_T1') == expected - expected = 'collection02/level-1/standard/oli-tirs/2019/157/021/LO08_L1GT_157021_20191221_20200924_02_T2/' \ - 'LO08_L1GT_157021_20191221_20200924_02_T2_stac.json' + expected = ( + 'collection02/level-1/standard/oli-tirs/2019/157/021/LO08_L1GT_157021_20191221_20200924_02_T2/' + 'LO08_L1GT_157021_20191221_20200924_02_T2_stac.json' + ) assert process.get_lc2_stac_json_key('LO08_L1GT_157021_20191221_20200924_02_T2') == expected - expected = 'collection02/level-1/standard/oli-tirs/2015/138/206/LT08_L1GT_138206_20150628_20200925_02_T2/' \ - 'LT08_L1GT_138206_20150628_20200925_02_T2_stac.json' + expected = ( + 'collection02/level-1/standard/oli-tirs/2015/138/206/LT08_L1GT_138206_20150628_20200925_02_T2/' + 'LT08_L1GT_138206_20150628_20200925_02_T2_stac.json' + ) assert process.get_lc2_stac_json_key('LT08_L1GT_138206_20150628_20200925_02_T2') == expected - expected = 'collection02/level-1/standard/etm/2006/024/035/LE07_L1TP_024035_20061119_20200913_02_T1/' \ - 'LE07_L1TP_024035_20061119_20200913_02_T1_stac.json' + expected = ( + 'collection02/level-1/standard/etm/2006/024/035/LE07_L1TP_024035_20061119_20200913_02_T1/' + 'LE07_L1TP_024035_20061119_20200913_02_T1_stac.json' + ) assert process.get_lc2_stac_json_key('LE07_L1TP_024035_20061119_20200913_02_T1') == expected - expected = 'collection02/level-1/standard/tm/1995/124/064/LT05_L1TP_124064_19950211_20200912_02_T1/' \ - 'LT05_L1TP_124064_19950211_20200912_02_T1_stac.json' + expected = ( + 'collection02/level-1/standard/tm/1995/124/064/LT05_L1TP_124064_19950211_20200912_02_T1/' + 'LT05_L1TP_124064_19950211_20200912_02_T1_stac.json' + ) assert process.get_lc2_stac_json_key('LT05_L1TP_124064_19950211_20200912_02_T1') == expected - expected = 'collection02/level-1/standard/mss/1995/098/068/LM05_L1GS_098068_19950831_20200823_02_T2/' \ - 'LM05_L1GS_098068_19950831_20200823_02_T2_stac.json' + expected = ( + 'collection02/level-1/standard/mss/1995/098/068/LM05_L1GS_098068_19950831_20200823_02_T2/' + 'LM05_L1GS_098068_19950831_20200823_02_T2_stac.json' + ) assert process.get_lc2_stac_json_key('LM05_L1GS_098068_19950831_20200823_02_T2') == expected - expected = 'collection02/level-1/standard/tm/1988/183/062/LT04_L1TP_183062_19880706_20200917_02_T1/' \ - 'LT04_L1TP_183062_19880706_20200917_02_T1_stac.json' + expected = ( + 'collection02/level-1/standard/tm/1988/183/062/LT04_L1TP_183062_19880706_20200917_02_T1/' + 'LT04_L1TP_183062_19880706_20200917_02_T1_stac.json' + ) assert process.get_lc2_stac_json_key('LT04_L1TP_183062_19880706_20200917_02_T1') == expected - expected = 'collection02/level-1/standard/mss/1983/117/071/LM04_L1GS_117071_19830609_20200903_02_T2/' \ - 'LM04_L1GS_117071_19830609_20200903_02_T2_stac.json' + expected = ( + 'collection02/level-1/standard/mss/1983/117/071/LM04_L1GS_117071_19830609_20200903_02_T2/' + 'LM04_L1GS_117071_19830609_20200903_02_T2_stac.json' + ) assert process.get_lc2_stac_json_key('LM04_L1GS_117071_19830609_20200903_02_T2') == expected @responses.activate def test_get_lc2_metadata(): responses.add( - responses.GET, f'{process.LC2_SEARCH_URL}/LC08_L1TP_009011_20200703_20200913_02_T1', - body='{"foo": "bar"}', status=200, + responses.GET, + f'{process.LC2_SEARCH_URL}/LC08_L1TP_009011_20200703_20200913_02_T1', + body='{"foo": "bar"}', + status=200, ) assert process.get_lc2_metadata('LC08_L1TP_009011_20200703_20200913_02_T1') == {'foo': 'bar'} @@ -101,9 +125,7 @@ def test_get_lc2_metadata_fallback(s3_stubber): 'Key': 'foo.json', 'RequestPayer': 'requester', } - s3_response = { - 'Body': io.StringIO('{"foo": "bar"}') - } + s3_response = {'Body': io.StringIO('{"foo": "bar"}')} s3_stubber.add_response(method='get_object', expected_params=params, service_response=s3_response) with mock.patch('hyp3_autorift.process.get_lc2_stac_json_key', return_value='foo.json'): @@ -111,21 +133,16 @@ def test_get_lc2_metadata_fallback(s3_stubber): def test_get_lc2_path(): - metadata = {'id': 'L--5', - 'assets': - {'B2.TIF': {'href': 'foo'}}} + metadata = {'id': 'L--5', 'assets': {'B2.TIF': {'href': 'foo'}}} assert process.get_lc2_path(metadata) == 'foo' - metadata = {'id': 'L--5', - 'assets': {'green': {'href': 'foo'}}} + metadata = {'id': 'L--5', 'assets': {'green': {'href': 'foo'}}} assert process.get_lc2_path(metadata) == 'foo' - metadata = {'id': 'L--8', - 'assets': {'B8.TIF': {'href': 'foo'}}} + metadata = {'id': 'L--8', 'assets': {'B8.TIF': {'href': 'foo'}}} assert process.get_lc2_path(metadata) == 'foo' - metadata = {'id': 'L--8', - 'assets': {'pan': {'href': 'foo'}}} + metadata = {'id': 'L--8', 'assets': {'pan': {'href': 'foo'}}} assert process.get_lc2_path(metadata) == 'foo' @@ -151,19 +168,25 @@ def test_get_s2_metadata(mock_get_s2_path: MagicMock, mock_get_raster_bbox: Magi def test_get_s2_safe_url(): - expected = 'https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/29/Q/KF/' \ - 'S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500.SAFE' + expected = ( + 'https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/29/Q/KF/' + 'S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500.SAFE' + ) assert process.get_s2_safe_url('S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500') == expected - expected = 'https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/38/E/MQ/' \ - 'S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056.SAFE' + expected = ( + 'https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/38/E/MQ/' + 'S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056.SAFE' + ) assert process.get_s2_safe_url('S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056') == expected @responses.activate def test_get_s2_manifest(): - url = 'https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/29/Q/KF/' \ - 'S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500.SAFE/manifest.safe' + url = ( + 'https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/29/Q/KF/' + 'S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500.SAFE/manifest.safe' + ) responses.add(responses.GET, url, body='foo', status=200) assert process.get_s2_manifest('S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500') == 'foo' @@ -180,41 +203,51 @@ def test_get_s2_path_aws(mock_s3_object_is_accessible: MagicMock): @patch('hyp3_autorift.process.s3_object_is_accessible') @patch('hyp3_autorift.process.get_s2_manifest') def test_get_s2_path_google_old_manifest( - mock_get_s2_manifest: MagicMock, mock_s3_object_is_accessible: MagicMock, test_data_directory: Path, + mock_get_s2_manifest: MagicMock, + mock_s3_object_is_accessible: MagicMock, + test_data_directory: Path, ): manifest = test_data_directory / 'S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500.manifest.safe' mock_get_s2_manifest.return_value = manifest.read_text() mock_s3_object_is_accessible.return_value = False path = process.get_s2_path('S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500') - assert path == '/vsicurl/https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/29/Q/KF/' \ - 'S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500.SAFE/./GRANULE' \ - '/S2A_OPER_MSI_L1C_TL_SGS__20160616T181414_A005139_T29QKF_N02.04/IMG_DATA' \ - '/S2A_OPER_MSI_L1C_TL_SGS__20160616T181414_A005139_T29QKF_B08.jp2' + assert ( + path == '/vsicurl/https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/29/Q/KF/' + 'S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500.SAFE/./GRANULE' + '/S2A_OPER_MSI_L1C_TL_SGS__20160616T181414_A005139_T29QKF_N02.04/IMG_DATA' + '/S2A_OPER_MSI_L1C_TL_SGS__20160616T181414_A005139_T29QKF_B08.jp2' + ) mock_get_s2_manifest.assert_called_once_with('S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500') mock_s3_object_is_accessible.assert_called_once_with( - 'its-live-project', 's2-cache/S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500_B08.jp2', + 'its-live-project', + 's2-cache/S2A_MSIL1C_20160616T112217_N0204_R137_T29QKF_20160617T193500_B08.jp2', ) @patch('hyp3_autorift.process.s3_object_is_accessible') @patch('hyp3_autorift.process.get_s2_manifest') def test_get_s2_path_google_new_manifest( - mock_get_s2_manifest: MagicMock, mock_s3_object_is_accessible: MagicMock, test_data_directory, + mock_get_s2_manifest: MagicMock, + mock_s3_object_is_accessible: MagicMock, + test_data_directory, ): manifest = test_data_directory / 'S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056.manifest.safe' mock_get_s2_manifest.return_value = manifest.read_text() mock_s3_object_is_accessible.return_value = False path = process.get_s2_path('S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056') - assert path == '/vsicurl/https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/38/E/MQ/' \ - 'S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056.SAFE/./GRANULE' \ - '/L1C_T38EMQ_A016290_20200419T060719/IMG_DATA/T38EMQ_20200419T060719_B08.jp2' + assert ( + path == '/vsicurl/https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/38/E/MQ/' + 'S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056.SAFE/./GRANULE' + '/L1C_T38EMQ_A016290_20200419T060719/IMG_DATA/T38EMQ_20200419T060719_B08.jp2' + ) mock_get_s2_manifest.assert_called_once_with('S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056') mock_s3_object_is_accessible.assert_called_once_with( - 'its-live-project', 's2-cache/S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056_B08.jp2', + 'its-live-project', + 's2-cache/S2B_MSIL1C_20200419T060719_N0209_R105_T38EMQ_20200419T091056_B08.jp2', ) @@ -242,10 +275,7 @@ def test_s3_object_is_accessible(s3_stubber): bucket = 'MyBucket' key = 'MyKey' - params = { - 'Bucket': bucket, - 'Key': key - } + params = {'Bucket': bucket, 'Key': key} s3_stubber.add_response(method='head_object', expected_params=params, service_response={}) assert process.s3_object_is_accessible(bucket, key) @@ -386,12 +416,16 @@ def test_point_to_prefix(): def test_get_lat_lon_from_ncfile(): - file = Path('tests/data/' - 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') + file = Path( + 'tests/data/' + 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc' + ) assert process.get_lat_lon_from_ncfile(file) == (-81.49, -128.28) def test_get_opendata_prefix(): - file = Path('tests/data/' - 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc') + file = Path( + 'tests/data/' + 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc' + ) assert process.get_opendata_prefix(file) == 'velocity_image_pair/landsatOLI/v02/S80W120' diff --git a/tests/test_s1_isce2.py b/tests/test_s1_isce2.py index 084253fa..a57dc855 100644 --- a/tests/test_s1_isce2.py +++ b/tests/test_s1_isce2.py @@ -4,14 +4,22 @@ def test_get_s1_primary_polarization(): - assert s1_isce2.get_s1_primary_polarization( - 'S1B_WV_SLC__1SSV_20200923T184541_20200923T185150_023506_02CA71_AABB') == 'vv' - assert s1_isce2.get_s1_primary_polarization( - 'S1B_IW_GRDH_1SDV_20200924T092954_20200924T093026_023515_02CABC_6C62') == 'vv' - assert s1_isce2.get_s1_primary_polarization( - 'S1B_IW_GRDH_1SSH_20200924T112903_20200924T112932_023516_02CAC7_D003') == 'hh' - assert s1_isce2.get_s1_primary_polarization( - 'S1B_IW_OCN__2SDH_20200924T090450_20200924T090519_023515_02CAB8_917B') == 'hh' + assert ( + s1_isce2.get_s1_primary_polarization('S1B_WV_SLC__1SSV_20200923T184541_20200923T185150_023506_02CA71_AABB') + == 'vv' + ) + assert ( + s1_isce2.get_s1_primary_polarization('S1B_IW_GRDH_1SDV_20200924T092954_20200924T093026_023515_02CABC_6C62') + == 'vv' + ) + assert ( + s1_isce2.get_s1_primary_polarization('S1B_IW_GRDH_1SSH_20200924T112903_20200924T112932_023516_02CAC7_D003') + == 'hh' + ) + assert ( + s1_isce2.get_s1_primary_polarization('S1B_IW_OCN__2SDH_20200924T090450_20200924T090519_023515_02CAB8_917B') + == 'hh' + ) with pytest.raises(ValueError): s1_isce2.get_s1_primary_polarization('S1A_EW_GRDM_1SHH_20150513T080355_20150513T080455_005900_007994_35D2') From 33aff57f7d80e45f50ac00e1aed6a2788c17f01b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 16 Dec 2024 13:39:38 -0500 Subject: [PATCH 06/11] exclude vendor code --- ruff.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ruff.toml b/ruff.toml index 61f91605..a6a36f22 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,3 +1,5 @@ +exclude = ["vend"] + line-length = 120 [format] From 222f47e232193908c9443d9c5913008b5653e311 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 16 Dec 2024 13:40:06 -0500 Subject: [PATCH 07/11] ruff in env --- environment.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 85f92caa..972fc59c 100644 --- a/environment.yml +++ b/environment.yml @@ -10,10 +10,7 @@ dependencies: - pip # For packaging, and testing - python-build - - flake8 - - flake8-import-order - - flake8-blind-except - - flake8-builtins + - ruff - pillow - pytest - pytest-console-scripts From 657d32ef2829c5facd46a1b1b0099132b8a9f2be Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 16 Dec 2024 11:26:12 -0900 Subject: [PATCH 08/11] pyproj ruff --- pyproject.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c07ae2a0..782dd4e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,7 @@ dynamic = ["version"] [project.optional-dependencies] develop = [ - 'flake8', - 'flake8-import-order', - 'flake8-blind-except', - 'flake8-builtins', + 'ruff', 'pillow', 'pytest', 'pytest-cov', From b124b1dfc145843e264ff9a744d9e2c49a4e64ff Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 17 Dec 2024 12:29:52 -0500 Subject: [PATCH 09/11] add new ruff config --- ruff.toml | 20 +++++++++++++++++++- src/hyp3_autorift/__init__.py | 1 + src/hyp3_autorift/crop.py | 1 + src/hyp3_autorift/geometry.py | 4 ++-- src/hyp3_autorift/image.py | 1 + src/hyp3_autorift/process.py | 1 + src/hyp3_autorift/s1_correction.py | 1 + src/hyp3_autorift/s1_isce2.py | 1 + 8 files changed, 27 insertions(+), 3 deletions(-) diff --git a/ruff.toml b/ruff.toml index a6a36f22..7b372679 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1,24 @@ exclude = ["vend"] - line-length = 120 +src = ["src", "tests"] [format] +indent-style = "space" quote-style = "single" + +[lint] +extend-select = [ + "I", # isort: https://docs.astral.sh/ruff/rules/#isort-i + # TODO: Uncomment the following extensions and address their warnings: + # "UP", # pyupgrade: https://docs.astral.sh/ruff/rules/#pyupgrade-up + # "D", # pydocstyle: https://docs.astral.sh/ruff/rules/#pydocstyle-d + # "ANN", # annotations: https://docs.astral.sh/ruff/rules/#flake8-annotations-ann + # "PTH", # use-pathlib-pth: https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth +] + +[lint.pydocstyle] +convention = "google" + +[lint.isort] +case-sensitive = true +lines-after-imports = 2 diff --git a/src/hyp3_autorift/__init__.py b/src/hyp3_autorift/__init__.py index c121b609..af9c0c7f 100644 --- a/src/hyp3_autorift/__init__.py +++ b/src/hyp3_autorift/__init__.py @@ -2,6 +2,7 @@ from importlib.metadata import PackageNotFoundError, version + try: __version__ = version(__name__) except PackageNotFoundError: diff --git a/src/hyp3_autorift/crop.py b/src/hyp3_autorift/crop.py index 5e2e8273..c227f1dc 100644 --- a/src/hyp3_autorift/crop.py +++ b/src/hyp3_autorift/crop.py @@ -36,6 +36,7 @@ import pyproj import xarray as xr + ENCODING_ATTRS = ['_FillValue', 'dtype', 'zlib', 'complevel', 'shuffle', 'add_offset', 'scale_factor'] diff --git a/src/hyp3_autorift/geometry.py b/src/hyp3_autorift/geometry.py index 97846a94..bb81859f 100644 --- a/src/hyp3_autorift/geometry.py +++ b/src/hyp3_autorift/geometry.py @@ -3,8 +3,8 @@ import logging from typing import Tuple -from osgeo import ogr -from osgeo import osr +from osgeo import ogr, osr + log = logging.getLogger(__name__) diff --git a/src/hyp3_autorift/image.py b/src/hyp3_autorift/image.py index eb901dac..5787ef5b 100644 --- a/src/hyp3_autorift/image.py +++ b/src/hyp3_autorift/image.py @@ -6,6 +6,7 @@ from matplotlib.colors import LinearSegmentedColormap from scipy.interpolate import PchipInterpolator + COLOR_MAP = np.array( [ # data value, R, G, B, A diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py index 6e1fd3ab..c4ee9154 100644 --- a/src/hyp3_autorift/process.py +++ b/src/hyp3_autorift/process.py @@ -24,6 +24,7 @@ from hyp3_autorift import geometry, image, utils from hyp3_autorift.crop import crop_netcdf_product + log = logging.getLogger(__name__) gdal.UseExceptions() diff --git a/src/hyp3_autorift/s1_correction.py b/src/hyp3_autorift/s1_correction.py index cc6f0470..dc60d054 100644 --- a/src/hyp3_autorift/s1_correction.py +++ b/src/hyp3_autorift/s1_correction.py @@ -7,6 +7,7 @@ from hyp3_autorift.process import DEFAULT_PARAMETER_FILE from hyp3_autorift.s1_isce2 import generate_correction_data + log = logging.getLogger(__name__) diff --git a/src/hyp3_autorift/s1_isce2.py b/src/hyp3_autorift/s1_isce2.py index 3daa8e78..4fe924b3 100644 --- a/src/hyp3_autorift/s1_isce2.py +++ b/src/hyp3_autorift/s1_isce2.py @@ -18,6 +18,7 @@ from hyp3_autorift import geometry, utils from hyp3_autorift.process import DEFAULT_PARAMETER_FILE + log = logging.getLogger(__name__) From e6c73d1160184ea2da8b90e4659e4461baacdcb5 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 17 Dec 2024 13:30:23 -0500 Subject: [PATCH 10/11] switch ruff config to pyproj --- pyproject.toml | 26 ++++++++++++++++++++++++++ ruff.toml | 24 ------------------------ 2 files changed, 26 insertions(+), 24 deletions(-) delete mode 100644 ruff.toml diff --git a/pyproject.toml b/pyproject.toml index 782dd4e1..8595e947 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,3 +63,29 @@ zip-safe = false where = ["src"] [tool.setuptools_scm] + +[tool.ruff] +exclude = ["vend"] +line-length = 120 +src = ["src", "tests"] + +[tool.ruff.format] +indent-style = "space" +quote-style = "single" + +[tool.ruff.lint] +extend-select = [ + "I", # isort: https://docs.astral.sh/ruff/rules/#isort-i + # TODO: Uncomment the following extensions and address their warnings: + # "UP", # pyupgrade: https://docs.astral.sh/ruff/rules/#pyupgrade-up + # "D", # pydocstyle: https://docs.astral.sh/ruff/rules/#pydocstyle-d + # "ANN", # annotations: https://docs.astral.sh/ruff/rules/#flake8-annotations-ann + # "PTH", # use-pathlib-pth: https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth +] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +case-sensitive = true +lines-after-imports = 2 diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index 7b372679..00000000 --- a/ruff.toml +++ /dev/null @@ -1,24 +0,0 @@ -exclude = ["vend"] -line-length = 120 -src = ["src", "tests"] - -[format] -indent-style = "space" -quote-style = "single" - -[lint] -extend-select = [ - "I", # isort: https://docs.astral.sh/ruff/rules/#isort-i - # TODO: Uncomment the following extensions and address their warnings: - # "UP", # pyupgrade: https://docs.astral.sh/ruff/rules/#pyupgrade-up - # "D", # pydocstyle: https://docs.astral.sh/ruff/rules/#pydocstyle-d - # "ANN", # annotations: https://docs.astral.sh/ruff/rules/#flake8-annotations-ann - # "PTH", # use-pathlib-pth: https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth -] - -[lint.pydocstyle] -convention = "google" - -[lint.isort] -case-sensitive = true -lines-after-imports = 2 From f927ad8f877ec9599cc54846004abb26652fdf04 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 17 Dec 2024 10:07:34 -0900 Subject: [PATCH 11/11] comment --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8595e947..fd20d057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,8 @@ where = ["src"] [tool.ruff] exclude = ["vend"] line-length = 120 +# The directories to consider when resolving first- vs. third-party imports. +# See: https://docs.astral.sh/ruff/settings/#src src = ["src", "tests"] [tool.ruff.format]