Merge pull request #216 from ASFHyP3/develop

Release v0.6.0 -- Restructuring

.github/workflows/changelog.yml

@@ -11,6 +11,6 @@ on:
       - develop
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.7.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.8.2
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/create-jira-issue.yml

@@ -0,0 +1,15 @@
+name: Create Jira issue
+
+on:
+  issues:
+    types: [labeled]
+
+jobs:
+  call-create-jira-issue-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
+    secrets:
+      JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
+      JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
+      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+      JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }}
+      JIRA_FIELDS: ${{ secrets.JIRA_FIELDS }}

.github/workflows/distribute.yml

@@ -7,27 +7,29 @@ on:
 
 jobs:
   call-version-info-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.7.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.8.2
 
   distribute:
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: mamba-org/provision-with-micromamba@v15
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
 
       - name: Build distribution
         id: build
         run: |
           python -m build
 
       - name: upload to PyPI.org
-        uses: pypa/gh-action-pypi-publish@v1.6.4
+        uses: pypa/gh-action-pypi-publish@v1.8.10
         with:
           user: __token__
           password: ${{ secrets.TOOLS_PYPI_PAK }}
@@ -41,9 +43,11 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: mamba-org/provision-with-micromamba@v15
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
 
       - name: Ensure asf_tools v${{ needs.call-version-info-workflow.outputs.version }}} is pip installable
         run: |

.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.7.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.8.2

.github/workflows/release.yml

@@ -9,7 +9,7 @@ jobs:
   finish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           token: ${{ secrets.TOOLS_BOT_PAK }}

.github/workflows/static-analysis.yml

@@ -6,7 +6,7 @@ jobs:
   flake8:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-python@v4
         with:
@@ -24,4 +24,4 @@ jobs:
 
 
   call-secrets-analysis-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.7.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.8.2

.github/workflows/tag-version.yml

@@ -7,6 +7,6 @@ on:
 
 jobs:
   call-bump-version-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.7.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.8.2
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}

.github/workflows/test-and-build.yml

@@ -12,16 +12,16 @@ on:
 
 jobs:
   call-pytest-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.7.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.8.2
     with:
       local_package_name: asf_tools
 
   call-version-info-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.7.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.8.2
 
   call-docker-ghcr-workflow:
     needs: call-version-info-workflow
-    uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.7.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.8.2
     with:
       version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }}
     secrets:

CHANGELOG.md

@@ -7,6 +7,31 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [0.6.0]
+
+## Added
+* You can choose whether the `ts` (threat score; default) or `fmi` (Fowlkes-Mallows index) minimization metric is used for the flood mapping iterative estimator:
+  * the `flood_map` console script entrypoint now accepts a `--minimization-metric` argument
+  * the  `asf_tools.hydrosar.floopd_map.make_flood_map` function now accepts a `minimization_metric` keyword argument
+* The flood mapping iterative estimator will ignore waterbodies smaller than a minimum number of pixels (default = 0)
+  * the `flood_map` console script entrypoint now accepts a `--iterative-min-size` argument
+  * the  `asf_tools.hydrosar.floopd_map.make_flood_map` function now accepts a `iterative_min_size` keyword argument
+
+### Changed
+* The HydroSAR code (`flood_map`, `water_map`, and `hand`) in `asf_tools` has been isolated to an `asf_tools.hydrosar` sub-package
+* The `asf_tools.hydrosar.flood_map.iterative` estimator now runs with a maximum step size of 3 instead of the default 0.5.
+* The `asf_tools.hydrosar.flood_map.iterative` estimator now uses the mean of the iterative bounds at the initial guess.
+* the known water threshold used to determine perennial water when creating flood maps will be calculated `asf_tools.hydrosar.flood_map.get_pw_threshold` if not provided
+* `get_epsg_code` and `epsg_to_wkt` have been moved from`asf_tools.composite` to `asf_tools.util`
+* `read_as_array` and `write_cog` have been moved from`asf_tools.composite` to `asf_tools.raster`
+* `get_coordinates` has been moved from`asf_tools.flood_map` to `asf_tools.util`
+
+### Deprecated
+* The `asf_tools.hydrosar` sub-package is being moved to the [HydroSAR project repository](https://github.com/fjmeyer/hydrosar) and will be provided in a new pip/conda installable package `hydrosar`. The `asf_tools.hydrosar` subpackage will be removed in a future release.
+
+### Fixed
+* Reverted the special handling of nan values introduced in v0.5.2, now that GDAL v3.7.0 has been released.
+
 ## [0.5.2]
 
 ### Added
@@ -20,13 +45,13 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [0.5.1]
 
 ### Changed
-* `asf-tools.flood_map` now produces rasters with pixel values of a positive integer where water is present and `0` where water is not present. Everywhere else is set to nodata.
+* `asf_tools.flood_map` now produces rasters with pixel values of a positive integer where water is present and `0` where water is not present. Everywhere else is set to nodata.
 
 ## [0.5.0]
 
 ### Added
 * HyP3 plugin entrypoints `water_map` and `flood_depth`
-  * Added fuzzy and intial VV and VH geotiffs back to water map output package.
+  * Added fuzzy and initial VV and VH geotiffs back to water map output package.
 * `asf_tools.__main__` entrypoint that allows you to select which hyp3 plugin entrypoint you'd like to run 
   (e.g., `python -m asf_tools ++process water_map ...`)
 
@@ -51,7 +76,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [0.4.4]
 
 ### Changed
-* `asf-tools.water_map` now produces water extent rasters with pixels values of `1` where water is present and `0` where water is not present
+* `asf_tools.water_map` now produces water extent rasters with pixels values of `1` where water is present and `0` where water is not present
 * `asf_tools.water_map` now uses the  updated [ASF Global HAND dataset](https://glo-hand-30m.s3.amazonaws.com/readme.html) derived from the [2021 release of the Copernicus GLO-30 Public DEM](https://spacedata.copernicus.eu/blogs/-/blogs/copernicus-dem-2021-release-now-available)
 
 ## [0.4.3]

environment.yml

@@ -22,7 +22,7 @@ dependencies:
   - astropy
   - boto3
   - fiona
-  - gdal>=3.3
+  - gdal>=3.7
   - numpy
   - pysheds>=0.3
   - rasterio

prototype/osl-env.yml

@@ -11,6 +11,8 @@ dependencies:
   - fiona
   - gdal>=3.2.0
   - hyp3_sdk>=1.1
+  - ipywidgets=7.7.3          # must match OSL base env!
+  - jupyterlab_widgets=1.1.2  # must match OSL base env!
   - jupyter
   - kernda
   - numpy

prototype/water-extent-map-on-demand.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# On Demand water maps via HyP3-watermap\n",
+    "# On Demand water extent and flood maps from HyP3-watermap\n",
     "\n",
     "This notebook will leverage either [ASF Search -- Vertex](https://search.asf.alaska.edu/#/) or the\n",
     "[asf_search](https://github.com/asfadmin/Discovery-asf_search) Python package, and the\n",
@@ -211,13 +211,23 @@
     "    'name': 'water-map-example',\n",
     "    'job_type': 'WATER_MAP',\n",
     "    'job_parameters': {\n",
-    "        'resolution': 30,\n",
-    "        'speckle_filter': True,\n",
-    "        'max_vv_threshold': -15.5,\n",
-    "        'max_vh_threshold': -23.0,\n",
-    "        'hand_threshold': 15.0,\n",
-    "        'hand_fraction': 0.8,\n",
-    "        'membership_threshold': 0.45,\n",
+    "        # RTC options\n",
+    "        'resolution': 30,               # Desired output pixel spacing in meters\n",
+    "        'speckle_filter': False,        # Apply an Enhanced Lee speckle filter\n",
+    "        # Water extent options\n",
+    "        'max_vv_threshold': -15.5,      # Maximum threshold value to use for VV polarized raster in decibels (dB)\n",
+    "        'max_vh_threshold': -23.0,      # Maximum threshold value to use for VH polarized raster in decibels (dB)\n",
+    "        'hand_threshold': 15.0,         # The maximum height above nearest drainage in meters to consider a pixel valid\n",
+    "        'hand_fraction': 0.8,           # The minimum fraction of valid HAND pixels required in a tile for thresholding\n",
+    "        'membership_threshold': 0.45,   # The average membership to the fuzzy indicators required for a water pixel\n",
+    "        # Flood  map options\n",
+    "        'flood_depth_estimator': None,  # iterative', 'logstat', 'nmad', 'numpy' or None; flood maps won't be calculated if None\n",
+    "        'water_level_sigma': 3.0,       # Standard deviation to estimate max water height for each object. Ignored when flood_depth_estimator is None\n",
+    "        'known_water_threshold': None,  # Threshold for extracting known water area in percent. Computed when None and ignored when flood_depth_estimator is None\n",
+    "        'iterative_min': 0,             # Minimum bound used for iterative estimator. Ignored when flood_depth_estimator is None\n",
+    "        'iterative_max': 15,            # Maximum bound used for iterative estimator. Ignored when flood_depth_estimator is None\n",
+    "        'iterative_min_size': 0,        # Minimum size of a connected waterbody in pixels for calculating flood depths with the iterative estimator\n",
+    "        'minimization_metric': 'ts',    # Minimization method when using the iterative estimator (only); Fowlkes-Mallows index (fmi) or a threat score (ts)\n",
     "    }\n",
     "}"
    ]
@@ -312,7 +322,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "asf-tools",
+   "display_name": "Python [conda env:.local-asf-tools]",
    "language": "python",
    "name": "conda-env-.local-asf-tools-py"
   },
@@ -326,9 +336,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.4"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
 }

pyproject.toml

@@ -38,13 +38,13 @@ dynamic = ["version"]
 
 [project.scripts]
 make_composite = "asf_tools.composite:main"
-water_map = "asf_tools.water_map:main"
-calculate_hand = "asf_tools.hand.calculate:main"
-flood_map = "asf_tools.flood_map:main"
+water_map = "asf_tools.hydrosar.water_map:main"
+calculate_hand = "asf_tools.hydrosar.hand.calculate:main"
+flood_map = "asf_tools.hydrosar.flood_map:main"
 
 [project.entry-points.hyp3]
-water_map = "asf_tools.water_map:hyp3"
-flood_map = "asf_tools.flood_map:hyp3"
+water_map = "asf_tools.hydrosar.water_map:hyp3"
+flood_map = "asf_tools.hydrosar.flood_map:hyp3"
 
 [project.optional-dependencies]
 develop = [
@@ -70,3 +70,8 @@ zip-safe = false
 where = ["src"]
 
 [tool.setuptools_scm]
+
+[tool.pytest.ini_options]
+markers = [
+    "integration: long-running integration tests",
+]

src/asf_tools/README.md

@@ -105,6 +105,13 @@ of the weights used to get the output average backscatter.
 
 ### Water extent mapping
 
+>[!WARNING]
+> The HydroSAR codes (`flood_map`, `water_map` and `hand` modules) are being
+> moved to the [HydroSAR project repository](https://github.com/fjmeyer/hydrosar)
+> and will be provided in a new pip/conda installable package `hydrosar`.
+> 
+> The `asf_tools.hydrosar` subpackage will be removed in a future release.
+
 The `water_map` tool allows you to create a surface water extent map from a Sentinel-1
 dual-pol (VV+VH) RTC product. It is intended to be used with RTC products generated by
 [ASF HyP3](https://hyp3-docs.asf.alaska.edu/using/vertex).
@@ -129,8 +136,12 @@ For details on the algorithm see the `asf_tools.water_map.make_water_map` docstr
 
 ### Flood depth mapping
 
-**Warning:** The flood depth tool is still under active development and the products
-created using this tool are likely to change in the future.
+>[!WARNING]
+> The HydroSAR codes (`flood_map`, `water_map` and `hand` modules) are being
+> moved to the [HydroSAR project repository](https://github.com/fjmeyer/hydrosar)
+> and will be provided in a new pip/conda installable package `hydrosar`.
+> 
+> The `asf_tools.hydrosar` subpackage will be removed in a future release.
 
 The `flood_map` tool allows you to create an estimated flood depth map from the surface
 water extent map created by the `water_map` tool.