From 576f4c4c2d9445b2ce2683ce6a169b0c084e518a Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 11:12:20 -0700 Subject: [PATCH 01/39] Adding a 2-phase docker build for testing. RE:#1431 --- docker/.gitignore | 1 + docker/Dockerfile | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 docker/.gitignore create mode 100644 docker/Dockerfile diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 0000000000..19b0cd20da --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1 @@ +environment.yml diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..f953be830d --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,19 @@ +# syntax=docker/dockerfile:1 + +# Build the InVEST wheel in a separate container stage +FROM debian:12.2 as build +RUN apt update && apt install python3 python3-dev python3-pip python3-build build-essential +RUN python3 -m build && pwd + +# Create the container for distribution that has runtime dependencies. +FROM mambaorg/micromamba:1.5.0-bookworm-slim +COPY --from build /dist/*.whl /tmp/ + +# The environment.yml file will be built during github actions. +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml +RUN micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ + micromamba clean --all --yes && \ + /opt/conda/bin/python -m pip install /tmp/*.whl + +ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] + From 495ea3239a83d901651c3cbb2e34dbc803e0ac43 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 11:17:01 -0700 Subject: [PATCH 02/39] Dockerfile syntaxerror RE:#1431 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f953be830d..874dc7c058 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ RUN python3 -m build && pwd # Create the container for distribution that has runtime dependencies. FROM mambaorg/micromamba:1.5.0-bookworm-slim -COPY --from build /dist/*.whl /tmp/ +COPY --from=build /dist/*.whl /tmp/ # The environment.yml file will be built during github actions. COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml From 8cac295fb90edc739e2f219e323ce0385ebad349 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 11:19:49 -0700 Subject: [PATCH 03/39] Installing apt packages with -y. RE:#1431 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 874dc7c058..7717721b8b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,7 @@ # Build the InVEST wheel in a separate container stage FROM debian:12.2 as build -RUN apt update && apt install python3 python3-dev python3-pip python3-build build-essential +RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential RUN python3 -m build && pwd # Create the container for distribution that has runtime dependencies. From b9d38db18df142ff8e4547762d80c12feca4d499 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 11:25:51 -0700 Subject: [PATCH 04/39] Allowing invest repo and build ref to be provided as build args. RE:#1431 --- docker/Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7717721b8b..7b85d79a47 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,13 +1,19 @@ # syntax=docker/dockerfile:1 +ARG INVEST_VERSION=main +ARG INVEST_REPO=natcap/invest + # Build the InVEST wheel in a separate container stage FROM debian:12.2 as build -RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential -RUN python3 -m build && pwd +RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git +RUN git clone https://github.com/${INVEST_REPO}.git && \ + cd invest && \ + git checkout ${INVEST_VERSION} && \ + python3 -m build # Create the container for distribution that has runtime dependencies. FROM mambaorg/micromamba:1.5.0-bookworm-slim -COPY --from=build /dist/*.whl /tmp/ +COPY --from=build /invest/dist/*.whl /tmp/ # The environment.yml file will be built during github actions. COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml From c9d72de019499ce9207a4ddff55611a9447e9214 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 11:29:49 -0700 Subject: [PATCH 05/39] Trying to quote the strings instead to try to get default arg values RE:#1431 --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7b85d79a47..0c78612e26 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 -ARG INVEST_VERSION=main -ARG INVEST_REPO=natcap/invest +ARG INVEST_VERSION="main" +ARG INVEST_REPO="natcap/invest" # Build the InVEST wheel in a separate container stage FROM debian:12.2 as build From 3195875b19c85cbb97aab043810b72c1b507e1d5 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 11:32:50 -0700 Subject: [PATCH 06/39] Build contexts are not shared between images. RE:#1431 --- docker/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0c78612e26..ab0bfcc53e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,9 @@ # syntax=docker/dockerfile:1 -ARG INVEST_VERSION="main" -ARG INVEST_REPO="natcap/invest" - # Build the InVEST wheel in a separate container stage FROM debian:12.2 as build +ARG INVEST_VERSION="main" +ARG INVEST_REPO="natcap/invest" RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git RUN git clone https://github.com/${INVEST_REPO}.git && \ cd invest && \ From 44d5b1e83266962fa464e8083bbac9b0fa1692e0 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 11:36:35 -0700 Subject: [PATCH 07/39] Adding venv, making dirname depend on repo. RE:#1431 --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ab0bfcc53e..9bc4702044 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,9 +4,9 @@ FROM debian:12.2 as build ARG INVEST_VERSION="main" ARG INVEST_REPO="natcap/invest" -RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git +RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv RUN git clone https://github.com/${INVEST_REPO}.git && \ - cd invest && \ + cd $(basename ${INVEST_REPO}) && \ git checkout ${INVEST_VERSION} && \ python3 -m build From 55cb3d36a445e9e491b5de1fa2075025ba04f214 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 11:47:45 -0700 Subject: [PATCH 08/39] Updating requirements and removing wheel after build. RE:#1431 --- docker/Dockerfile | 3 ++- requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9bc4702044..985e220881 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,8 @@ COPY --from=build /invest/dist/*.whl /tmp/ COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml RUN micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ micromamba clean --all --yes && \ - /opt/conda/bin/python -m pip install /tmp/*.whl + /opt/conda/bin/python -m pip install /tmp/*.whl && \ + rm /tmp/*.whl ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] diff --git a/requirements.txt b/requirements.txt index 47dd9a47b8..a2a0151546 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,8 +17,8 @@ numpy>=1.11.0,!=1.16.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 scipy>=1.9.0 -pygeoprocessing==2.4.0 # pip-only -taskgraph[niced_processes]>=0.11.0 # pip-only +pygeoprocessing==2.4.0 +taskgraph>=0.11.0 psutil>=5.6.6 chardet>=3.0.4 pint From a90a79563bf36e9deefe3ed5e4e017a5fec4d99b Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 12:51:14 -0700 Subject: [PATCH 09/39] Adding a docker container workflow building job. RE:#1431 --- .github/workflows/build-docker.yml | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/build-docker.yml diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000000..c5eb851c76 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,66 @@ +name: Build containers + +on: + workflow_dispatch: + push: + branches: + - 'main' + paths: + - 'docker/*' + - '.github/workflows/build-docker.yml' + - 'src/**/*' + +concurrency: + # make sure only one run of this workflow for a given PR or a given branch + # can happen at one time. previous queued or started runs will be cancelled. + # github.workflow is the workflow name + # github.ref is the ref that triggered the workflow run + # on push, this is refs/heads/ + # on pull request, this is refs/pull//merge + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -l {0} + +env: + # Customize this name if needed. + # The repo name is a very reasonable default! + CONTAINER_NAME: invest + +jobs: + build: + name: Build containers + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - uses: actions/checkout@v3 + + - name: login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Determine version + run: | + echo "DATESTAMP_VERSION=$(date +%F)" >> $GITHUB_ENV + + - name: Build docker + run: | + cd docker && docker build \ + -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ + -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha}} \ + -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.ref_name}} \ + . + + - name: Test that GDAL and InVEST import + run: | + docker run --rm ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest python -c "from osgeo import gdal" + docker run --rm ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest python -m natcap.invest --version + + - name: Push docker + run: docker image push --all-tags ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }} From efa1df2738669c4effa4bc5d52cc749f60b8d01f Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 12:53:23 -0700 Subject: [PATCH 10/39] Creating the environment file. RE:#1431 --- .github/workflows/build-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index c5eb851c76..4feb1025c3 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -45,9 +45,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Determine version + - name: Create environment file run: | - echo "DATESTAMP_VERSION=$(date +%F)" >> $GITHUB_ENV + python scripts/convert-requirements-to-conda-yml.py requirements.txt > docker/environment.yml - name: Build docker run: | From 6bfd67f048c62efcfc5fa49e44d4c421c10513dc Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 12:56:01 -0700 Subject: [PATCH 11/39] Removing the branch restriction for building the container. RE:#1431 --- .github/workflows/build-docker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 4feb1025c3..f30e01d9c4 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -3,8 +3,6 @@ name: Build containers on: workflow_dispatch: push: - branches: - - 'main' paths: - 'docker/*' - '.github/workflows/build-docker.yml' From bb642676e605edce5ff85635f49c5b8f7b5612b8 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 12:59:38 -0700 Subject: [PATCH 12/39] Sanitizing the github ref. RE:#1431 --- .github/workflows/build-docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index f30e01d9c4..8dfeca41c5 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -49,10 +49,12 @@ jobs: - name: Build docker run: | + # Replace / (invalid character) with . + SANITIZED_REF="${{ github.ref_name }//\//.}" cd docker && docker build \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha}} \ - -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.ref_name}} \ + -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${SANITIZED_REF} \ . - name: Test that GDAL and InVEST import From a3f951f4d9da3a0d592bfb2a041684ea6f541884 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 13:01:46 -0700 Subject: [PATCH 13/39] Using sed for substring replacement instead of bash builtings. GHA doesn't like all the nested curly braces. RE:#1431 --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 8dfeca41c5..02cd1c22fd 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -50,7 +50,7 @@ jobs: - name: Build docker run: | # Replace / (invalid character) with . - SANITIZED_REF="${{ github.ref_name }//\//.}" + SANITIZED_REF="$(echo ${{github.ref_name}} | sed 's|/|.|g')" cd docker && docker build \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha}} \ From 77be2d44268158346b8c055fcea9faf691af925b Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 13:07:41 -0700 Subject: [PATCH 14/39] Not removing the wheel, not writeable. RE:#1431 --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 985e220881..9bc4702044 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,8 +18,7 @@ COPY --from=build /invest/dist/*.whl /tmp/ COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml RUN micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ micromamba clean --all --yes && \ - /opt/conda/bin/python -m pip install /tmp/*.whl && \ - rm /tmp/*.whl + /opt/conda/bin/python -m pip install /tmp/*.whl ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] From beea25d7fea96d432af96cc77fa7a652f202372e Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 13:12:46 -0700 Subject: [PATCH 15/39] Adding requests to requirements. RE:#1431 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index a2a0151546..d9dc564d7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,4 @@ pint Babel Flask flask_cors +requests From 781789d2100a0c9521a584c228cb8a86612c75ad Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 13:13:50 -0700 Subject: [PATCH 16/39] Adding requirements to the list of triggering files. RE:#1431 --- .github/workflows/build-docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 02cd1c22fd..9f8aa02475 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -7,6 +7,7 @@ on: - 'docker/*' - '.github/workflows/build-docker.yml' - 'src/**/*' + - 'requirements*.txt' concurrency: # make sure only one run of this workflow for a given PR or a given branch From 5f5817e48c7f38d4f03121237c718bbd55c3d86c Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 13:46:18 -0700 Subject: [PATCH 17/39] Adding note about the docker container to HISTORY. RE:#1431 --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 59523f494b..71cf40551b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,9 @@ Unreleased Changes have been replaced with ``numpy.prod``. https://github.com/natcap/invest/issues/1410 * Add support for python 3.11 (`#1103 `_) + * Adding a docker container that is built on each commit where a change to + model code, requirements, or the docker configuration has been made. + https://github.com/natcap/invest/issues/1115 * NDR * Fixing an issue where minor geometric issues in the watersheds input (such as a ring self-intersection) would raise an error in the model. From c181d28e58d4ff479adbebb86ae3f556f5ceb122 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 14:26:27 -0700 Subject: [PATCH 18/39] Building latest pygeoprocessing and installing into container. RE:#1431 --- docker/Dockerfile | 10 +++++++++- requirements.txt | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9bc4702044..b3a71ef534 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,14 +5,22 @@ FROM debian:12.2 as build ARG INVEST_VERSION="main" ARG INVEST_REPO="natcap/invest" RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv -RUN git clone https://github.com/${INVEST_REPO}.git && \ +RUN cd / && \ + git clone https://github.com/${INVEST_REPO}.git && \ cd $(basename ${INVEST_REPO}) && \ git checkout ${INVEST_VERSION} && \ python3 -m build +RUN cd / && \ + git clone https://github.com/natcap/pygeoprocessing.git && \ + cd pygeoprocessing && \ + git checkout main && \ + python3 -m build + # Create the container for distribution that has runtime dependencies. FROM mambaorg/micromamba:1.5.0-bookworm-slim COPY --from=build /invest/dist/*.whl /tmp/ +COPY --from=build /pygeoprocessing/dist/*.whl /tmp/ # The environment.yml file will be built during github actions. COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml diff --git a/requirements.txt b/requirements.txt index d9dc564d7e..0f1b680e30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ numpy>=1.11.0,!=1.16.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 scipy>=1.9.0 -pygeoprocessing==2.4.0 +pygeoprocessing # grab latest during development taskgraph>=0.11.0 psutil>=5.6.6 chardet>=3.0.4 From f6ea5ec86477126e4c24b57bdaccdccfe09c8db7 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 14:36:20 -0700 Subject: [PATCH 19/39] Passing invest repo and version to the build image. RE:#1431 --- .github/workflows/build-docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 9f8aa02475..5c6db38035 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -53,8 +53,10 @@ jobs: # Replace / (invalid character) with . SANITIZED_REF="$(echo ${{github.ref_name}} | sed 's|/|.|g')" cd docker && docker build \ + --build-arg="INVEST_REPO=${{ github.repository }}" \ + --build-arg="INVEST_VERSION=${{ github.sha }}" \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \ - -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha}} \ + -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha }} \ -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${SANITIZED_REF} \ . From eab9919749a00c8e5458c42afb03153e7b0ba905 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 15:32:15 -0700 Subject: [PATCH 20/39] Sediment deposition now uses more memory, but can handle larger indexes. RE:#1431 --- src/natcap/invest/sdr/sdr_core.pyx | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/natcap/invest/sdr/sdr_core.pyx b/src/natcap/invest/sdr/sdr_core.pyx index ada579f54a..c6848bfcf1 100644 --- a/src/natcap/invest/sdr/sdr_core.pyx +++ b/src/natcap/invest/sdr/sdr_core.pyx @@ -43,9 +43,6 @@ cdef int *ROW_OFFSETS = [0, -1, -1, -1, 0, 1, 1, 1] cdef int *COL_OFFSETS = [1, 1, 0, -1, -1, -1, 0, 1] -cdef int is_close(double x, double y): - return abs(x-y) <= (1e-8+1e-05*abs(y)) - # this is a least recently used cache written in C++ in an external file, # exposing here so _ManagedRaster can use it cdef extern from "LRUCache.h" nogil: @@ -435,7 +432,7 @@ def calculate_sediment_deposition( # is the original pixel `x` cdef int *inflow_offsets = [4, 5, 6, 7, 0, 1, 2, 3] - cdef int n_cols, n_rows + cdef long n_cols, n_rows flow_dir_info = pygeoprocessing.get_raster_info(mfd_flow_direction_path) n_cols, n_rows = flow_dir_info['raster_size'] cdef int mfd_nodata = 0 @@ -444,11 +441,12 @@ def calculate_sediment_deposition( sdr_path)['nodata'][0] cdef float e_prime_nodata = pygeoprocessing.get_raster_info( e_prime_path)['nodata'][0] - cdef int col_index, row_index, win_xsize, win_ysize, xoff, yoff - cdef int global_col, global_row, flat_index, j, k - cdef int seed_col = 0 - cdef int seed_row = 0 - cdef int neighbor_row, neighbor_col + cdef long col_index, row_index, win_xsize, win_ysize, xoff, yoff + cdef long global_col, global_row, j, k + cdef unsigned long flat_index + cdef long seed_col = 0 + cdef long seed_row = 0 + cdef long neighbor_row, neighbor_col cdef int flow_val, neighbor_flow_val, ds_neighbor_flow_val cdef int flow_weight, neighbor_flow_weight cdef float flow_sum, neighbor_flow_sum From 2d6cc1e2dfc3c3284f1af45152676e84ed2b4118 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 15:34:56 -0700 Subject: [PATCH 21/39] Noting change in HISTORY for SDR. RE:#1431 --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 71cf40551b..a2d00cde1c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -55,6 +55,9 @@ Unreleased Changes * SDR * RKLS, USLE, avoided erosion, and avoided export rasters will now have nodata in streams (`#1415 `_) + * Fixed an issue in SDR's sediment deposition where, on rasters with more + than 2^31 pixels, the model would crash with an error relating to a + negative (overflowed) index. https://github.com/natcap/invest/issues/1431 * Wind Energy * Fixed a bug where model would error when the grid points path was empty (`#1417 `_) From 0d647f33b499109454cbb3ed6fd2c529cf99da14 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 15:40:07 -0700 Subject: [PATCH 22/39] Improving the number of pixels that can be included in the analysis. This comes at a near-doubling of required stack memory (and some heap memory), but it should be OK for most users. RE:#1431 --- HISTORY.rst | 3 +++ src/natcap/invest/ndr/ndr_core.pyx | 17 +++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a2d00cde1c..f9ae5ab2b8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -52,6 +52,9 @@ Unreleased Changes * Fixed a task dependency issue where NDR would crash because of a race condition when run with ``n_workers > 0``. https://github.com/natcap/invest/issues/1426 + * Fixed an issue in NDR's effective retention where, on rasters with more + than 2^31 pixels, the model would crash with an error relating to a + negative (overflowed) index. https://github.com/natcap/invest/issues/1431 * SDR * RKLS, USLE, avoided erosion, and avoided export rasters will now have nodata in streams (`#1415 `_) diff --git a/src/natcap/invest/ndr/ndr_core.pyx b/src/natcap/invest/ndr/ndr_core.pyx index 11ed14296e..b56ba40d12 100644 --- a/src/natcap/invest/ndr/ndr_core.pyx +++ b/src/natcap/invest/ndr/ndr_core.pyx @@ -390,11 +390,11 @@ def ndr_eff_calculation( cdef int *col_offsets = [1, 1, 0, -1, -1, -1, 0, 1] cdef int *inflow_offsets = [4, 5, 6, 7, 0, 1, 2, 3] - cdef int n_cols, n_rows + cdef long n_cols, n_rows flow_dir_info = pygeoprocessing.get_raster_info(mfd_flow_direction_path) n_cols, n_rows = flow_dir_info['raster_size'] - cdef stack[int] processing_stack + cdef stack[long] processing_stack stream_info = pygeoprocessing.get_raster_info(stream_path) # cell sizes must be square, so no reason to test at this point. cdef float cell_size = abs(stream_info['pixel_size'][0]) @@ -427,13 +427,14 @@ def ndr_eff_calculation( cdef _ManagedRaster to_process_flow_directions_raster = _ManagedRaster( to_process_flow_directions_path, 1, True) - cdef int col_index, row_index, win_xsize, win_ysize, xoff, yoff - cdef int global_col, global_row - cdef int flat_index, outflow_weight, outflow_weight_sum, flow_dir - cdef int ds_col, ds_row, i + cdef long col_index, row_index, win_xsize, win_ysize, xoff, yoff + cdef long global_col, global_row + cdef unsigned long flat_index + cdef long outflow_weight, outflow_weight_sum, flow_dir + cdef long ds_col, ds_row, i cdef float current_step_factor, step_size, crit_len - cdef int neighbor_row, neighbor_col, neighbor_outflow_dir - cdef int neighbor_outflow_dir_mask, neighbor_process_flow_dir + cdef long neighbor_row, neighbor_col + cdef int neighbor_outflow_dir, neighbor_outflow_dir_mask, neighbor_process_flow_dir cdef int outflow_dirs, dir_mask for offset_dict in pygeoprocessing.iterblocks( From 6acc0a7636ca683574e5d157ab39272b27e30a85 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 16:02:34 -0700 Subject: [PATCH 23/39] Upping stack/heap memory usage for indexes in seasonal water yield's cython core. RE:#1431 --- HISTORY.rst | 5 ++++ .../seasonal_water_yield_core.pyx | 28 +++++++++++-------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f9ae5ab2b8..a5a939f19d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -61,6 +61,11 @@ Unreleased Changes * Fixed an issue in SDR's sediment deposition where, on rasters with more than 2^31 pixels, the model would crash with an error relating to a negative (overflowed) index. https://github.com/natcap/invest/issues/1431 +* Seasonal Water Yield + * Fixed an issue in Seasonal Water Yield's baseflow routing and local + recharge functions where, on rasters with more than 2^31 pixels, the + model would crash with an error relating to a negative (overflowed) + index. https://github.com/natcap/invest/issues/1431 * Wind Energy * Fixed a bug where model would error when the grid points path was empty (`#1417 `_) diff --git a/src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx b/src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx index 8b26896ecc..6c6aad9eae 100644 --- a/src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx +++ b/src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx @@ -415,17 +415,20 @@ cpdef calculate_local_recharge( """ cdef int i_n, flow_dir_nodata, flow_dir_mfd cdef int peak_pixel - cdef int xs, ys, xs_root, ys_root, xoff, yoff, flow_dir_s - cdef int xi, yi, xj, yj, flow_dir_j, p_ij_base - cdef int win_xsize, win_ysize, n_dir - cdef int raster_x_size, raster_y_size + cdef long xs, ys, xs_root, ys_root, xoff, yoff + cdef int flow_dir_s + cdef long xi, yi, xj, yj + cdef int flow_dir_j, p_ij_base + cdef long win_xsize, win_ysize + cdef int n_dir + cdef long raster_x_size, raster_y_size cdef double pet_m, p_m, qf_m, et0_m, aet_i, p_i, qf_i, l_i, l_avail_i cdef float qf_nodata, kc_nodata cdef int j_neighbor_end_index, mfd_dir_sum cdef float mfd_direction_array[8] - cdef queue[pair[int, int]] work_queue + cdef queue[pair[long, long]] work_queue cdef _ManagedRaster et0_m_raster, qf_m_raster, kc_m_raster cdef numpy.ndarray[numpy.npy_float32, ndim=1] alpha_month_array = ( @@ -554,7 +557,7 @@ cpdef calculate_local_recharge( break if peak_pixel: work_queue.push( - pair[int, int](xs_root, ys_root)) + pair[long, long](xs_root, ys_root)) while work_queue.size() > 0: xi = work_queue.front().first @@ -685,7 +688,7 @@ cpdef calculate_local_recharge( if (xi_n < 0 or xi_n >= raster_x_size or yi_n < 0 or yi_n >= raster_y_size): continue - work_queue.push(pair[int, int](xi_n, yi_n)) + work_queue.push(pair[long, long](xi_n, yi_n)) def route_baseflow_sum( @@ -717,13 +720,14 @@ def route_baseflow_sum( cdef float target_nodata = -1e32 cdef int stream_val, outlet cdef float b_i, b_sum_i, l_j, l_avail_j, l_sum_j - cdef int xi, yi, xj, yj, flow_dir_i, p_ij_base + cdef long xi, yi, xj, yj + cdef int flow_dir_i, p_ij_base cdef int mfd_dir_sum, flow_dir_nodata - cdef int raster_x_size, raster_y_size, xs_root, ys_root, xoff, yoff + cdef long raster_x_size, raster_y_size, xs_root, ys_root, xoff, yoff cdef int n_dir cdef int xs, ys, flow_dir_s, win_xsize, win_ysize cdef int stream_nodata - cdef stack[pair[int, int]] work_stack + cdef stack[pair[long, long]] work_stack # we know the PyGeoprocessing MFD raster flow dir type is a 32 bit int. flow_dir_raster_info = pygeoprocessing.get_raster_info(flow_dir_mfd_path) @@ -783,7 +787,7 @@ def route_baseflow_sum( break if not outlet: continue - work_stack.push(pair[int, int](xs_root, ys_root)) + work_stack.push(pair[long, long](xs_root, ys_root)) while work_stack.size() > 0: xi = work_stack.top().first @@ -871,4 +875,4 @@ def route_baseflow_sum( if (0xF & (flow_dir_j >> ( 4 * FLOW_DIR_REVERSE_DIRECTION[n_dir]))): # pixel flows here, push on queue - work_stack.push(pair[int, int](xj, yj)) + work_stack.push(pair[long, long](xj, yj)) From db1a05e0a6794c44f4b17d7ffe22db40b14676fd Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 19 Oct 2023 16:21:12 -0700 Subject: [PATCH 24/39] Using the latest pygeoprocessing commit on main. RE:#1431 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0f1b680e30..6518b65fd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ numpy>=1.11.0,!=1.16.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 scipy>=1.9.0 -pygeoprocessing # grab latest during development +git+https://github.com/natcap/pygeoprocessing.git@main#egg=pygeoprocessing # pip-only taskgraph>=0.11.0 psutil>=5.6.6 chardet>=3.0.4 From 274f669719c33b6c58a7c3f49987930d6e5fb926 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Mon, 23 Oct 2023 18:31:17 -0700 Subject: [PATCH 25/39] Monitoring a few other paths for rebuilding docker image. RE:#1431 --- .github/workflows/build-docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 5c6db38035..254dc31940 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -8,6 +8,8 @@ on: - '.github/workflows/build-docker.yml' - 'src/**/*' - 'requirements*.txt' + - 'pyproject.toml' + - 'setup.py' concurrency: # make sure only one run of this workflow for a given PR or a given branch @@ -50,7 +52,7 @@ jobs: - name: Build docker run: | - # Replace / (invalid character) with . + # Replace / (invalid tag character) with . SANITIZED_REF="$(echo ${{github.ref_name}} | sed 's|/|.|g')" cd docker && docker build \ --build-arg="INVEST_REPO=${{ github.repository }}" \ From e6f850cb76bba6a56a7e5a7e7416de51d23311aa Mon Sep 17 00:00:00 2001 From: James Douglass Date: Mon, 23 Oct 2023 18:40:29 -0700 Subject: [PATCH 26/39] Replacing native pygeoprocessing builds with better yml generation. This way pip will perform scm-based package installation like normal with whatever extra dependencies it needs. RE:#1431 --- docker/Dockerfile | 7 ------- scripts/convert-requirements-to-conda-yml.py | 20 +++++++++++++++++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b3a71ef534..f8526d658a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,16 +11,9 @@ RUN cd / && \ git checkout ${INVEST_VERSION} && \ python3 -m build -RUN cd / && \ - git clone https://github.com/natcap/pygeoprocessing.git && \ - cd pygeoprocessing && \ - git checkout main && \ - python3 -m build - # Create the container for distribution that has runtime dependencies. FROM mambaorg/micromamba:1.5.0-bookworm-slim COPY --from=build /invest/dist/*.whl /tmp/ -COPY --from=build /pygeoprocessing/dist/*.whl /tmp/ # The environment.yml file will be built during github actions. COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml diff --git a/scripts/convert-requirements-to-conda-yml.py b/scripts/convert-requirements-to-conda-yml.py index 8d30efb172..b4a145f431 100644 --- a/scripts/convert-requirements-to-conda-yml.py +++ b/scripts/convert-requirements-to-conda-yml.py @@ -1,8 +1,8 @@ # encoding=UTF-8 """convert-requirements-to-conda-yml.py""" -import sys import argparse +import sys YML_TEMPLATE = """channels: - conda-forge @@ -46,8 +46,7 @@ def build_environment_from_requirements(cli_args): requirements_files = args.req pip_requirements = set() - # conda likes it when you list pip if you're using pip. - conda_requirements = set(['pip']) + conda_requirements = set() for requirement_file in requirements_files: with open(requirement_file) as file: for line in file: @@ -58,7 +57,22 @@ def build_environment_from_requirements(cli_args): continue if line.endswith('# pip-only'): + # Conda prefers that we explicitly include pip as a + # requirement if we're using pip. + conda_requirements.add('pip') + pip_requirements.add(line) + + # If git needs to be installed for pip to clone to a + # revision, add it to the conda package list. + # Bazaar (bzr, which pip supports) is not listed; + # deprecated as of 2016 and not available on conda-forge. + for prefix, conda_pkg in [("git+", "git"), + ("hg+", "mercurial"), + ("svn+", "subversion")]: + if line.startswith(prefix): + conda_requirements.add(conda_pkg) + break # only 1 of these can be true. else: conda_requirements.add(line) From 39ebcb35a27fec426dc3e6e03458fdcc5875035b Mon Sep 17 00:00:00 2001 From: James Douglass Date: Mon, 23 Oct 2023 18:43:04 -0700 Subject: [PATCH 27/39] Clearing pip cache during the build. RE:#1431 --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f8526d658a..e369589a69 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,8 @@ COPY --from=build /invest/dist/*.whl /tmp/ COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml RUN micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ micromamba clean --all --yes && \ - /opt/conda/bin/python -m pip install /tmp/*.whl + /opt/conda/bin/python -m pip install /tmp/*.whl && \ + /opt/conda/bin/python -m pip cache purge ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] From 65fd13881ec306485ae8fd6e5e44b15bbe9c35af Mon Sep 17 00:00:00 2001 From: James Douglass Date: Mon, 23 Oct 2023 18:57:09 -0700 Subject: [PATCH 28/39] Rebuilding container if conda yml script changes. RE#1431 --- .github/workflows/build-docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 254dc31940..b9bdfca7cd 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -10,6 +10,7 @@ on: - 'requirements*.txt' - 'pyproject.toml' - 'setup.py' + - 'scripts/convert-requirements-to-conda-yml.py' concurrency: # make sure only one run of this workflow for a given PR or a given branch From f7119e831db8a7d890469cc1e50323f524baf17d Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 11:38:37 -0700 Subject: [PATCH 29/39] Adding a compiler when we are building from source. RE:#1431 --- scripts/convert-requirements-to-conda-yml.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/convert-requirements-to-conda-yml.py b/scripts/convert-requirements-to-conda-yml.py index b4a145f431..c069d709ff 100644 --- a/scripts/convert-requirements-to-conda-yml.py +++ b/scripts/convert-requirements-to-conda-yml.py @@ -63,16 +63,23 @@ def build_environment_from_requirements(cli_args): pip_requirements.add(line) - # If git needs to be installed for pip to clone to a + # If an scm needs to be installed for pip to clone to a # revision, add it to the conda package list. + # # Bazaar (bzr, which pip supports) is not listed; # deprecated as of 2016 and not available on conda-forge. - for prefix, conda_pkg in [("git+", "git"), - ("hg+", "mercurial"), - ("svn+", "subversion")]: + for prefix, scm_conda_pkg in [("git+", "git"), + ("hg+", "mercurial"), + ("svn+", "subversion")]: if line.startswith(prefix): - conda_requirements.add(conda_pkg) - break # only 1 of these can be true. + conda_requirements.add(scm_conda_pkg) + # Always make the compiler available + # cxx-compiler works on all OSes. + # NOTE: do not use this as a dependency in a + # conda-forge package recipe! + # https://anaconda.org/conda-forge/cxx-compiler + conda_requirements.add('cxx-compiler') + break # The line can only match 1 prefix else: conda_requirements.add(line) From ce7c59d7d9debc86537753b07986eb1086a2c5eb Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 12:08:00 -0700 Subject: [PATCH 30/39] Removing cxx-compiler and git after build completes. RE#1431 --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e369589a69..aebdcd9fab 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,8 @@ COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml RUN micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ micromamba clean --all --yes && \ /opt/conda/bin/python -m pip install /tmp/*.whl && \ - /opt/conda/bin/python -m pip cache purge + /opt/conda/bin/python -m pip cache purge && \ + micromamba remove -y cxx-compiler git ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] From 4800f4a98f7d02ffa1cdc970389f9264562ad581 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 12:17:22 -0700 Subject: [PATCH 31/39] Specifying the base prefix. RE:#1431 --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index aebdcd9fab..ab38b0a139 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,7 +21,6 @@ RUN micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \ micromamba clean --all --yes && \ /opt/conda/bin/python -m pip install /tmp/*.whl && \ /opt/conda/bin/python -m pip cache purge && \ - micromamba remove -y cxx-compiler git + micromamba remove -y -n base cxx-compiler git ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] - From 8bfd8d0d1b50bad5731905ca68022c6626a71b14 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 12:26:59 -0700 Subject: [PATCH 32/39] Attempting to address a build warning about setuptools_scm/setuptools version incompatibility that I encountered in github actions. RE:#1431 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c3f19787a6..fed1937a6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ invest = "natcap.invest.cli:main" # that we can provide a much easier build experience so long as GDAL is # available at runtime. requires = [ - 'setuptools>=45', 'wheel', 'setuptools_scm>=6.4.0', 'cython', 'babel', + 'setuptools>=61', 'wheel', 'setuptools_scm>=8.0', 'cython', 'babel', 'oldest-supported-numpy' ] build-backend = "setuptools.build_meta" From 4c82149a4594aa44638eed314af9c56a42f19242 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 12:30:11 -0700 Subject: [PATCH 33/39] Allowing git lfs install to fail. When running on github actions, I found an error noting that `Hook already exists: pre-push` (see below), which to me makes me think that LFS already exists and has been configured. And I confirm that on GHA, git-lfs is installed to version 3.4.0 ``` git -C data/invest-test-data lfs install Hook already exists: pre-push #!/bin/sh command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; } git lfs pre-push "$@" To resolve this, either: 1: run `git lfs update --manual` for instructions on how to merge hooks. 2: run `git lfs update --force` to overwrite your hook. ``` --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 39514077a0..9d57de71b4 100644 --- a/Makefile +++ b/Makefile @@ -196,7 +196,7 @@ $(GIT_UG_REPO_PATH): $(GIT_SAMPLE_DATA_REPO_PATH): | $(DATA_DIR) -git clone $(GIT_SAMPLE_DATA_REPO) $(GIT_SAMPLE_DATA_REPO_PATH) git -C $(GIT_SAMPLE_DATA_REPO_PATH) fetch - git -C $(GIT_SAMPLE_DATA_REPO_PATH) lfs install + -git -C $(GIT_SAMPLE_DATA_REPO_PATH) lfs install git -C $(GIT_SAMPLE_DATA_REPO_PATH) checkout $(GIT_SAMPLE_DATA_REPO_REV) git -C $(GIT_SAMPLE_DATA_REPO_PATH) lfs fetch git -C $(GIT_SAMPLE_DATA_REPO_PATH) lfs checkout @@ -204,7 +204,7 @@ $(GIT_SAMPLE_DATA_REPO_PATH): | $(DATA_DIR) $(GIT_TEST_DATA_REPO_PATH): | $(DATA_DIR) -git clone $(GIT_TEST_DATA_REPO) $(GIT_TEST_DATA_REPO_PATH) git -C $(GIT_TEST_DATA_REPO_PATH) fetch - git -C $(GIT_TEST_DATA_REPO_PATH) lfs install + -git -C $(GIT_TEST_DATA_REPO_PATH) lfs install git -C $(GIT_TEST_DATA_REPO_PATH) checkout $(GIT_TEST_DATA_REPO_REV) git -C $(GIT_TEST_DATA_REPO_PATH) lfs fetch git -C $(GIT_TEST_DATA_REPO_PATH) lfs checkout From 8301c75e930c89ae31eff5590f2c94c18457efc6 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 13:20:45 -0700 Subject: [PATCH 34/39] Restoring pygeoprocessing 2.4.0 requirement. RE:#1431 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6518b65fd9..ade366a618 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ numpy>=1.11.0,!=1.16.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 scipy>=1.9.0 -git+https://github.com/natcap/pygeoprocessing.git@main#egg=pygeoprocessing # pip-only +pygeoprocessing==2.4.0 # pip-only taskgraph>=0.11.0 psutil>=5.6.6 chardet>=3.0.4 From 76c84ba0e41e5b3582bb02ff20e4f36f684c5fc3 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 13:31:52 -0700 Subject: [PATCH 35/39] Explicitly prohibiting uploading to the registry when part of a PR, just in case. RE:#1431 --- .github/workflows/build-docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index b9bdfca7cd..47be5d8625 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -69,4 +69,5 @@ jobs: docker run --rm ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest python -m natcap.invest --version - name: Push docker + if: github.event_name != 'pull_request' run: docker image push --all-tags ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }} From ab8400fef9357f9058768e8bd102ef4c57cfc877 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 14:08:56 -0700 Subject: [PATCH 36/39] Removing the pip constraint on pygeoprocessing. RE:#1431 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ade366a618..d9dc564d7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ numpy>=1.11.0,!=1.16.0 Rtree>=0.8.2,!=0.9.1 shapely>=2.0.0 scipy>=1.9.0 -pygeoprocessing==2.4.0 # pip-only +pygeoprocessing==2.4.0 taskgraph>=0.11.0 psutil>=5.6.6 chardet>=3.0.4 From 89d840afb411f908ea8d2330b3c00f05f8893f92 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 24 Oct 2023 14:11:22 -0700 Subject: [PATCH 37/39] Adding a compiler if on linux or it's needed. A compiler will be needed to build from source. RE:#1431 --- scripts/convert-requirements-to-conda-yml.py | 21 ++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/scripts/convert-requirements-to-conda-yml.py b/scripts/convert-requirements-to-conda-yml.py index c069d709ff..e5e2bc2d57 100644 --- a/scripts/convert-requirements-to-conda-yml.py +++ b/scripts/convert-requirements-to-conda-yml.py @@ -2,6 +2,7 @@ """convert-requirements-to-conda-yml.py""" import argparse +import platform import sys YML_TEMPLATE = """channels: @@ -68,18 +69,26 @@ def build_environment_from_requirements(cli_args): # # Bazaar (bzr, which pip supports) is not listed; # deprecated as of 2016 and not available on conda-forge. + install_compiler = False for prefix, scm_conda_pkg in [("git+", "git"), ("hg+", "mercurial"), ("svn+", "subversion")]: if line.startswith(prefix): conda_requirements.add(scm_conda_pkg) - # Always make the compiler available - # cxx-compiler works on all OSes. - # NOTE: do not use this as a dependency in a - # conda-forge package recipe! - # https://anaconda.org/conda-forge/cxx-compiler - conda_requirements.add('cxx-compiler') + install_compiler = True break # The line can only match 1 prefix + + # It's less common (like for pygeoprocessing) to have linux + # wheels. Install a compiler if we're on linux, to be + # safe. + if platform.system() == 'Linux' or install_compiler: + # Always make the compiler available + # cxx-compiler works on all OSes. + # NOTE: do not use this as a dependency in a + # conda-forge package recipe! + # https://anaconda.org/conda-forge/cxx-compiler + conda_requirements.add('cxx-compiler') + else: conda_requirements.add(line) From d9aeced9b7e10d60103c1dfca8866ae6fe63aea1 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Wed, 25 Oct 2023 09:35:38 -0700 Subject: [PATCH 38/39] Fixing history message heading. RE:#1431 --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 7f3b8f42cf..522ab8477a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -65,6 +65,7 @@ Unreleased Changes * Fixed an issue in SDR's sediment deposition where, on rasters with more than 2^31 pixels, the model would crash with an error relating to a negative (overflowed) index. https://github.com/natcap/invest/issues/1431 +* Seasonal Water Yield * Fixed an issue in Seasonal Water Yield's baseflow routing and local recharge functions where, on rasters with more than 2^31 pixels, the model would crash with an error relating to a negative (overflowed) From cd492aa4b0c79555259d7a4f114681c3517dbf05 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Wed, 25 Oct 2023 09:41:47 -0700 Subject: [PATCH 39/39] Removing failure-case handling for git lfs install. RE:#1431 --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9d57de71b4..39514077a0 100644 --- a/Makefile +++ b/Makefile @@ -196,7 +196,7 @@ $(GIT_UG_REPO_PATH): $(GIT_SAMPLE_DATA_REPO_PATH): | $(DATA_DIR) -git clone $(GIT_SAMPLE_DATA_REPO) $(GIT_SAMPLE_DATA_REPO_PATH) git -C $(GIT_SAMPLE_DATA_REPO_PATH) fetch - -git -C $(GIT_SAMPLE_DATA_REPO_PATH) lfs install + git -C $(GIT_SAMPLE_DATA_REPO_PATH) lfs install git -C $(GIT_SAMPLE_DATA_REPO_PATH) checkout $(GIT_SAMPLE_DATA_REPO_REV) git -C $(GIT_SAMPLE_DATA_REPO_PATH) lfs fetch git -C $(GIT_SAMPLE_DATA_REPO_PATH) lfs checkout @@ -204,7 +204,7 @@ $(GIT_SAMPLE_DATA_REPO_PATH): | $(DATA_DIR) $(GIT_TEST_DATA_REPO_PATH): | $(DATA_DIR) -git clone $(GIT_TEST_DATA_REPO) $(GIT_TEST_DATA_REPO_PATH) git -C $(GIT_TEST_DATA_REPO_PATH) fetch - -git -C $(GIT_TEST_DATA_REPO_PATH) lfs install + git -C $(GIT_TEST_DATA_REPO_PATH) lfs install git -C $(GIT_TEST_DATA_REPO_PATH) checkout $(GIT_TEST_DATA_REPO_REV) git -C $(GIT_TEST_DATA_REPO_PATH) lfs fetch git -C $(GIT_TEST_DATA_REPO_PATH) lfs checkout