From c77f71638336e52ad1a66bcad232cb05cf39714c Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 10 Oct 2024 17:39:50 +0700 Subject: [PATCH 01/57] adding dev container config --- .devcontainer/devcontainer.json | 9 +++++++ .devcontainer/setup.sh | 6 +++++ .devcontainer/welcome-message.txt | 13 +++++++++ .github/.devcontainer/devcontainer.json | 33 +++++++++++++++++++++++ .github/workflows/devcontainer.yaml | 36 +++++++++++++++++++++++++ Dockerfile | 18 +++++++++++++ Dockerfile.solved | 26 ++++++++++++++++++ config.tutorial.yaml | 1 + 8 files changed, 142 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/setup.sh create mode 100644 .devcontainer/welcome-message.txt create mode 100644 .github/.devcontainer/devcontainer.json create mode 100644 .github/workflows/devcontainer.yaml create mode 100644 Dockerfile create mode 100644 Dockerfile.solved diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..81e1605d9 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,9 @@ +// For format details, see https://aka.ms/devcontainer.json. +{ + "name": "pypsa earth dev", + "image": "ghcr.io/drifter089/pypsa-earth:latest", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces,type=bind,consistency=cached", + "initializeCommand": "docker pull ghcr.io/drifter089/pypsa-earth:latest", + "workspaceFolder": "/workspaces", + "postAttachCommand": "bash .devcontainer/setup.sh" +} diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100644 index 000000000..e2fc1a3e8 --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# mv ../pypsa-eur/* ./ +# echo 'solved tutorial files are copied to workspace' + +cat .devcontainer/welcome-message.txt \ No newline at end of file diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt new file mode 100644 index 000000000..a0c311068 --- /dev/null +++ b/.devcontainer/welcome-message.txt @@ -0,0 +1,13 @@ + + +👋 Welcome to pypsa-eur contribution in a dev container! + Works in VS Code, or in docker using the devcontainer cli + +See https://github.com/drifter089/pypsa-eur/blob/master/README.md for guidance on debugging and contributing to pypsa-eur + + +For details about dev containers see https://containers.dev + +The configuration for the dev container is in the .github/.devcontainer folder. + +🔍 To explore VS Code to its fullest, search using the Command Palette (Cmd/Ctrl + Shift + P or F1).+ \ No newline at end of file diff --git a/.github/.devcontainer/devcontainer.json b/.github/.devcontainer/devcontainer.json new file mode 100644 index 000000000..fda35213c --- /dev/null +++ b/.github/.devcontainer/devcontainer.json @@ -0,0 +1,33 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu +{ + "name": "pypsa earth dev", + "build": { + "dockerfile": "../../Dockerfile" + }, + + "features": { + "ghcr.io/devcontainers-contrib/features/bash-command:1": {}, + "ghcr.io/eliises/devcontainer-features/bash-profile:1": {} + }, + "customizations": { + "vscode": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/bash" + } + }, + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-azuretools.vscode-docker", + "ms-toolsai.jupyter", + "zainchen.json", + "tomoki1207.pdf", + "grapecity.gc-excelviewer" + ] + } + }, + + "postCreateCommand": "python -m pip install --upgrade debugpy" +} diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml new file mode 100644 index 000000000..ec48a223e --- /dev/null +++ b/.github/workflows/devcontainer.yaml @@ -0,0 +1,36 @@ +name: Dev Container Build and Push Image + +on: + workflow_dispatch: + push: + branches: + - "main" + tags: + - "v*.*.*" + pull_request: + branches: [main] + + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - + name: Checkout + id: checkout + uses: actions/checkout@v1 + - + name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.REGISTRY_TOKEN }} + - + name: Pre-build dev container image + uses: devcontainers/ci@v0.2 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }} + push: always \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..6c8c64296 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM condaforge/mambaforge + +RUN conda update -n base conda +RUN conda install -n base conda-libmamba-solver +RUN conda config --set solver libmamba + +WORKDIR /pypsa-earth + +COPY ./envs ./envs + +RUN conda env create --file envs/environment.yaml + +RUN rm -r envs + +RUN echo "source activate pypsa-earth" > ~/.bashrc +ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH + +ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/Dockerfile.solved b/Dockerfile.solved new file mode 100644 index 000000000..f832c7a2b --- /dev/null +++ b/Dockerfile.solved @@ -0,0 +1,26 @@ +FROM condaforge/mambaforge + +RUN conda update -n base conda +RUN conda install -n base conda-libmamba-solver +RUN conda config --set solver libmamba + +RUN apt-get update && apt-get install -y git + +WORKDIR /pypsa-earth + +COPY . . + +RUN conda env create --file envs/environment.yaml + +RUN echo "source activate pypsa-earth" > ~/.bashrc +ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH + +RUN /bin/bash -c "source activate pypsa-earth && snakemake -j 1 solve_all_networks" + +RUN git ls-files -z | xargs -0 rm + +RUN find . -type d -empty -delete + +RUN rm -rf scripts config data .snakemake .git + +ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 0c98bb152..02ef557aa 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,6 +9,7 @@ tutorial: true countries: ["NG", "BJ"] enable: + retrieve_databundle: true build_natura_raster: true progress_bar: false From b076ea3620b3970b204d6e910de893b9b9cc3639 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 10 Oct 2024 11:13:36 +0000 Subject: [PATCH 02/57] updated docekr file --- Dockerfile | 4 ++-- Dockerfile.solved | 26 -------------------------- config.tutorial.yaml | 1 - 3 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 Dockerfile.solved diff --git a/Dockerfile b/Dockerfile index 6c8c64296..200dfd4ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,11 +8,11 @@ WORKDIR /pypsa-earth COPY ./envs ./envs -RUN conda env create --file envs/environment.yaml +RUN conda env create -n pypsa-earth --file envs/environment.yaml RUN rm -r envs -RUN echo "source activate pypsa-earth" > ~/.bashrc +RUN echo "conda activate pypsa-earth" > ~/.bashrc ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/Dockerfile.solved b/Dockerfile.solved deleted file mode 100644 index f832c7a2b..000000000 --- a/Dockerfile.solved +++ /dev/null @@ -1,26 +0,0 @@ -FROM condaforge/mambaforge - -RUN conda update -n base conda -RUN conda install -n base conda-libmamba-solver -RUN conda config --set solver libmamba - -RUN apt-get update && apt-get install -y git - -WORKDIR /pypsa-earth - -COPY . . - -RUN conda env create --file envs/environment.yaml - -RUN echo "source activate pypsa-earth" > ~/.bashrc -ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH - -RUN /bin/bash -c "source activate pypsa-earth && snakemake -j 1 solve_all_networks" - -RUN git ls-files -z | xargs -0 rm - -RUN find . -type d -empty -delete - -RUN rm -rf scripts config data .snakemake .git - -ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 02ef557aa..0c98bb152 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,7 +9,6 @@ tutorial: true countries: ["NG", "BJ"] enable: - retrieve_databundle: true build_natura_raster: true progress_bar: false From 96d333b3714cd28ceca6a2c0b046fd3cedd47bf8 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 10 Oct 2024 19:58:13 +0700 Subject: [PATCH 03/57] updated docker file --- Dockerfile | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 200dfd4ed..ba8bd26e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,15 +4,25 @@ RUN conda update -n base conda RUN conda install -n base conda-libmamba-solver RUN conda config --set solver libmamba +RUN apt-get update && apt-get install -y bash + WORKDIR /pypsa-earth -COPY ./envs ./envs +COPY ./envs ./temp + +RUN conda env create -n pypsa-earth -f temp/environment.yaml + +RUN conda init bash -RUN conda env create -n pypsa-earth --file envs/environment.yaml +RUN touch ~/.bashrc && echo "conda activate pypsa-earth" >> ~/.bashrc -RUN rm -r envs +SHELL ["/bin/bash", "--login", "-c"] -RUN echo "conda activate pypsa-earth" > ~/.bashrc ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH -ENTRYPOINT ["tail", "-f", "/dev/null"] +RUN rm -r temp + +RUN conda clean -afy && \ + rm -rf /tmp/* + +CMD ["bash"] From 151bc2d11bcd13a64a4d4c80a1c5018013dc525a Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 14 Oct 2024 11:52:18 +0100 Subject: [PATCH 04/57] update: include java --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index ba8bd26e8..6f5b099d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,8 @@ SHELL ["/bin/bash", "--login", "-c"] ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH +RUN conda install conda-forge::openjdk -y + RUN rm -r temp RUN conda clean -afy && \ From 05cf1c49c93fec0529165da105746576525022c1 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 14 Oct 2024 21:12:49 +0100 Subject: [PATCH 05/57] docker docs init --- doc/docker_containers.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/docker_containers.rst diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst new file mode 100644 index 000000000..556ef61ba --- /dev/null +++ b/doc/docker_containers.rst @@ -0,0 +1,7 @@ +# Running PyPSA-Earth in Docker containers + +## How to use Docker containers to run PyPSA-Earth. +- install Docker +- git clone the pypsa-earth repository +- open the pypsa-earth with VSCode +- rebuild and open the repo in a container \ No newline at end of file From 0d65aac21c8e863cc72e33b8d8bed23e27240d39 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 14 Oct 2024 21:17:14 +0100 Subject: [PATCH 06/57] review welcome message for new users --- .devcontainer/welcome-message.txt | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt index a0c311068..9fe33b7d6 100644 --- a/.devcontainer/welcome-message.txt +++ b/.devcontainer/welcome-message.txt @@ -1,13 +1,21 @@ -👋 Welcome to pypsa-eur contribution in a dev container! - Works in VS Code, or in docker using the devcontainer cli -See https://github.com/drifter089/pypsa-eur/blob/master/README.md for guidance on debugging and contributing to pypsa-eur +👋 Welcome to the PyPSA-Earth Development Environment! -For details about dev containers see https://containers.dev +We’re excited to have you here! This setup allows you to contribute to PyPSA-Earth effortlessly using a development container in VS Code. -The configuration for the dev container is in the .github/.devcontainer folder. +📖 Getting Started for New Users -🔍 To explore VS Code to its fullest, search using the Command Palette (Cmd/Ctrl + Shift + P or F1).+ \ No newline at end of file + • For a step-by-step guide on setting up your environment, debugging, and making your first contribution, refer to the PyPSA-Earth README. It covers everything you need to know as a newcomer. + • The configuration files for the development container are located in the .github/.devcontainer folder. + +💡 Tips for New Users + + • Make the most of VS Code by using the Command Palette (Cmd/Ctrl + Shift + P or F1) for quick access to features and commands. + • If you’re new to development containers, learn the basics at containers.dev. + +🚀 Start Exploring and Happy Coding! + +Don’t hesitate to reach out if you need help—our community is here to support you. \ No newline at end of file From a3d22ca92c7bfdb4c7af678f5228426984a4d8ee Mon Sep 17 00:00:00 2001 From: drifter089 Date: Mon, 21 Oct 2024 16:12:35 +0700 Subject: [PATCH 07/57] running tutorial before push --- .github/.devcontainer/test.sh | 5 +++++ .github/workflows/devcontainer.yaml | 23 ++++++++++++++++------- Dockerfile | 2 +- config.tutorial.yaml | 3 ++- 4 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 .github/.devcontainer/test.sh diff --git a/.github/.devcontainer/test.sh b/.github/.devcontainer/test.sh new file mode 100644 index 000000000..c4ac3f290 --- /dev/null +++ b/.github/.devcontainer/test.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cp config.tutorial.yaml config.yaml + +snakemake -j 1 solve_all_networks \ No newline at end of file diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml index ec48a223e..1d689e233 100644 --- a/.github/workflows/devcontainer.yaml +++ b/.github/workflows/devcontainer.yaml @@ -27,10 +27,19 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.REGISTRY_TOKEN }} - - name: Pre-build dev container image - uses: devcontainers/ci@v0.2 - with: - subFolder: .github - imageName: ghcr.io/${{ github.repository }} - cacheFrom: ghcr.io/${{ github.repository }} - push: always \ No newline at end of file + name: Build Dev Container Image + uses: devcontainers/ci@v0.2 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }} + push: false + + - name: Test Container Image + run: | + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone https://github.com/drifter089/pypsa-earth/tree/devContainers && cd pypsa-earth && bash .github/.devcontainer/test.sh" + + - name: Push Container Image + if: ${{ success() }} + run: | + docker push ghcr.io/${{ github.repository }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6f5b099d6..343661abd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ RUN conda update -n base conda RUN conda install -n base conda-libmamba-solver RUN conda config --set solver libmamba -RUN apt-get update && apt-get install -y bash +RUN apt-get update && apt-get install -y bash git WORKDIR /pypsa-earth diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 0c98bb152..fb48057c9 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -6,9 +6,10 @@ version: 0.4.1 tutorial: true -countries: ["NG", "BJ"] +countries: ["NG"] enable: + retrieve_databundle: true build_natura_raster: true progress_bar: false From 952583a5cd53d1520c1066774e7194bb1a1b440f Mon Sep 17 00:00:00 2001 From: drifter089 Date: Mon, 21 Oct 2024 16:25:09 +0700 Subject: [PATCH 08/57] fixing clone command in CI --- .github/workflows/devcontainer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml index 1d689e233..67c172da8 100644 --- a/.github/workflows/devcontainer.yaml +++ b/.github/workflows/devcontainer.yaml @@ -37,7 +37,7 @@ jobs: - name: Test Container Image run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone https://github.com/drifter089/pypsa-earth/tree/devContainers && cd pypsa-earth && bash .github/.devcontainer/test.sh" + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - name: Push Container Image if: ${{ success() }} From 9a1fb06323a9cd43dfe379898ebb75f9df28af8e Mon Sep 17 00:00:00 2001 From: drifter089 Date: Mon, 21 Oct 2024 17:04:58 +0700 Subject: [PATCH 09/57] fixing variable value --- .github/workflows/devcontainer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml index 67c172da8..173b2b252 100644 --- a/.github/workflows/devcontainer.yaml +++ b/.github/workflows/devcontainer.yaml @@ -33,7 +33,7 @@ jobs: subFolder: .github imageName: ghcr.io/${{ github.repository }} cacheFrom: ghcr.io/${{ github.repository }} - push: false + push: never - name: Test Container Image run: | From ad189c5391cc87e3cbb19483509912ad1d82354f Mon Sep 17 00:00:00 2001 From: Yerbol Akhmetov <113768325+yerbol-akhmetov@users.noreply.github.com> Date: Wed, 16 Oct 2024 01:22:50 +0500 Subject: [PATCH 10/57] Fix issue of adding H2 store in `add_extra_components` rule (#1134) * fix issue of adding H2 store and make H2 Store in AC buses * add release notes * Add H2 Stores to both AC and DC buses --- doc/release_notes.rst | 2 ++ scripts/add_extra_components.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 1ea2f5393..470e9743d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -26,6 +26,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `_ +* Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `_ + PyPSA-Earth 0.4.1 ================= diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index cab8195df..2c317c305 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -104,7 +104,7 @@ def attach_stores(n, costs, config): _add_missing_carriers_from_costs(n, costs, carriers) - buses_i = n.buses.query("carrier == 'AC'").index + buses_i = n.buses.index bus_sub_dict = {k: n.buses[k].values for k in ["x", "y", "country"]} if "H2" in carriers: From fffe9c796a433b68200964972e1ffddd252fa13f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:24:48 +0200 Subject: [PATCH 11/57] [pre-commit.ci] pre-commit autoupdate (#1131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/psf/black: 24.8.0 → 24.10.0](https://github.com/psf/black/compare/24.8.0...24.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9fd50966b..54c3bcd16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ exclude: ^(LICENSES) repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer @@ -49,7 +49,7 @@ repos: # Formatting with "black" coding style - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 24.10.0 hooks: # Format Python files - id: black From ed8e300dcc1a7217edddbf7259c3b5f2a0267aa5 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Wed, 16 Oct 2024 12:56:09 +0200 Subject: [PATCH 12/57] Add a restriction to rasterio version (#1146) --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index d98f772c3..7da885a73 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -56,7 +56,7 @@ dependencies: # GIS dependencies: - cartopy - descartes -- rasterio!=1.2.10 +- rasterio!=1.2.10, <=1.3.11 - rioxarray # Plotting From 95e0aab64bdfa906663a937af8ad18828ba2418a Mon Sep 17 00:00:00 2001 From: Yerbol Akhmetov <113768325+yerbol-akhmetov@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:39:47 +0500 Subject: [PATCH 13/57] Enable configfile specification for mock_snakemake (#1135) * enable configfile for mock_snakemake * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add release notes * drop else condition --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- doc/release_notes.rst | 2 ++ scripts/_helpers.py | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 470e9743d..a900e5860 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -28,6 +28,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `_ +* Enable configfile specification for mock_snakemake `PR #1135 `_ + PyPSA-Earth 0.4.1 ================= diff --git a/scripts/_helpers.py b/scripts/_helpers.py index e3ceb2adf..76bf0268d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -522,7 +522,9 @@ def get_aggregation_strategies(aggregation_strategies): return bus_strategies, generator_strategies -def mock_snakemake(rulename, root_dir=None, submodule_dir=None, **wildcards): +def mock_snakemake( + rulename, root_dir=None, submodule_dir=None, configfile=None, **wildcards +): """ This function is expected to be executed from the "scripts"-directory of " the snakemake project. It returns a snakemake.script.Snakemake object, @@ -534,6 +536,8 @@ def mock_snakemake(rulename, root_dir=None, submodule_dir=None, **wildcards): ---------- rulename: str name of the rule for which the snakemake object should be generated + configfile: str + path to config file to be used in mock_snakemake wildcards: keyword arguments fixing the wildcards. Only necessary if wildcards are needed. @@ -566,9 +570,17 @@ def mock_snakemake(rulename, root_dir=None, submodule_dir=None, **wildcards): if os.path.exists(p): snakefile = p break + + if isinstance(configfile, str): + with open(configfile, "r") as file: + configfile = yaml.safe_load(file) + workflow = sm.Workflow( - snakefile, overwrite_configfiles=[], rerun_triggers=[] - ) # overwrite_config=config + snakefile, + overwrite_configfiles=[], + rerun_triggers=[], + overwrite_config=configfile, + ) workflow.include(snakefile) workflow.global_resources = {} try: From fb5c61ba73307a86365c670694c07fb47e8f6375 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Tue, 22 Oct 2024 12:06:23 +0100 Subject: [PATCH 14/57] update: set env file required for cluster network --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 343661abd..b2b171658 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,4 +27,7 @@ RUN rm -r temp RUN conda clean -afy && \ rm -rf /tmp/* +ENV LD_PRELOAD=/opt/conda/envs/pypsa-earth/lib/python3.10/site-packages/sklearn/utils/../../../../libgomp.so.1 + CMD ["bash"] + From 4fc5fc362a8240ff38c5345c846d1156ff46335f Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 28 Oct 2024 15:21:59 +0100 Subject: [PATCH 15/57] update: include mac into the CI --- .../{devcontainer.yaml => devcontainer.yml} | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) rename .github/workflows/{devcontainer.yaml => devcontainer.yml} (88%) diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yml similarity index 88% rename from .github/workflows/devcontainer.yaml rename to .github/workflows/devcontainer.yml index 173b2b252..a41bf6f4c 100644 --- a/.github/workflows/devcontainer.yaml +++ b/.github/workflows/devcontainer.yml @@ -13,9 +13,17 @@ on: jobs: build-and-push: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: + - [ubuntu-latest] + - [macos-latest] + + runs-on: ${{ matrix.os }} + steps: - - + - name: Checkout id: checkout uses: actions/checkout@v1 From b54eacd5a86848c0d5df4a6602874db3374cc827 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:00:08 +0700 Subject: [PATCH 16/57] adding user as secret --- .github/workflows/devcontainer.yaml | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/devcontainer.yaml diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml new file mode 100644 index 000000000..d38dd1fe0 --- /dev/null +++ b/.github/workflows/devcontainer.yaml @@ -0,0 +1,45 @@ +name: Dev Container Build and Push Image + +on: + workflow_dispatch: + push: + branches: + - "main" + tags: + - "v*.*.*" + pull_request: + branches: [main] + + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - + name: Checkout + id: checkout + uses: actions/checkout@v1 + - + name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + - + name: Build Dev Container Image + uses: devcontainers/ci@v0.2 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }} + push: never + + - name: Test Container Image + run: | + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" + + - name: Push Container Image + if: ${{ success() }} + run: | + docker push ghcr.io/${{ github.repository }} \ No newline at end of file From 5a6e21008917fd040752fe01fd4a294e527c7581 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:11:33 +0700 Subject: [PATCH 17/57] delete duplicate file and add user as secret --- .github/workflows/devcontainer.yaml | 45 ----------------------------- .github/workflows/devcontainer.yml | 2 +- 2 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 .github/workflows/devcontainer.yaml diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml deleted file mode 100644 index d38dd1fe0..000000000 --- a/.github/workflows/devcontainer.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Dev Container Build and Push Image - -on: - workflow_dispatch: - push: - branches: - - "main" - tags: - - "v*.*.*" - pull_request: - branches: [main] - - -jobs: - build-and-push: - runs-on: ubuntu-latest - steps: - - - name: Checkout - id: checkout - uses: actions/checkout@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Build Dev Container Image - uses: devcontainers/ci@v0.2 - with: - subFolder: .github - imageName: ghcr.io/${{ github.repository }} - cacheFrom: ghcr.io/${{ github.repository }} - push: never - - - name: Test Container Image - run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - - - name: Push Container Image - if: ${{ success() }} - run: | - docker push ghcr.io/${{ github.repository }} \ No newline at end of file diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index a41bf6f4c..16ab1c038 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -32,7 +32,7 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image From 8abaa7b2560b055fd4d7d5dc90cb7a6296ebbaca Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:21:57 +0700 Subject: [PATCH 18/57] setup docekr on macos --- .github/workflows/devcontainer.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 16ab1c038..71db12c9b 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -27,13 +27,17 @@ jobs: name: Checkout id: checkout uses: actions/checkout@v1 + + - name: Setup Docker on macOS + if: runner.os == 'macOS' + uses: douglascamata/setup-docker-macos-action@v1-alpha - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_TOKEN }} + name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image uses: devcontainers/ci@v0.2 From a2f570e9bc975c9e59781f6086d59191601bcd20 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:24:57 +0700 Subject: [PATCH 19/57] trying colima for docekr setup on mac --- .github/workflows/devcontainer.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 71db12c9b..b5b1d4f7c 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -28,9 +28,13 @@ jobs: id: checkout uses: actions/checkout@v1 - - name: Setup Docker on macOS + - name: Setup Colima (Docker alternative for macOS) if: runner.os == 'macOS' - uses: douglascamata/setup-docker-macos-action@v1-alpha + run: | + brew install colima docker + colima start --cpu 2 --memory 4 + # Wait for Docker to be ready + while ! docker info >/dev/null 2>&1; do sleep 1; done - name: Login to GitHub Container Registry uses: docker/login-action@v2 From 9f4a0bfe095bf6ee2ca184fe9cb0b81c9a5121c1 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:33:37 +0700 Subject: [PATCH 20/57] adding mac docker setup --- .github/workflows/devcontainer.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index b5b1d4f7c..dd2e49843 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -18,7 +18,7 @@ jobs: matrix: os: - [ubuntu-latest] - - [macos-latest] + - [ macos-13 ] runs-on: ${{ matrix.os }} @@ -28,15 +28,13 @@ jobs: id: checkout uses: actions/checkout@v1 - - name: Setup Colima (Docker alternative for macOS) - if: runner.os == 'macOS' + - name: Setup docker (missing on MacOS) + if: runner.os == 'macos' run: | - brew install colima docker - colima start --cpu 2 --memory 4 - # Wait for Docker to be ready - while ! docker info >/dev/null 2>&1; do sleep 1; done - - - name: Login to GitHub Container Registry + brew install docker + colima start + + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io From 928195e520890b0f9a0deaf0293c7e25eb6bb4de Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:38:06 +0700 Subject: [PATCH 21/57] installing colima on mac --- .github/workflows/devcontainer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index dd2e49843..8de44c3a8 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -32,6 +32,7 @@ jobs: if: runner.os == 'macos' run: | brew install docker + brew install colima colima start - name: Login to GitHub Container Registry From d50db33d9eeb20d0e5e3657c1198ec420367d787 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:54:32 +0700 Subject: [PATCH 22/57] disable buildkit for docker --- .github/workflows/devcontainer.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 8de44c3a8..d1ae2402a 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -43,6 +43,8 @@ jobs: password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image + env: + DOCKER_BUILDKIT: ${{ runner.os == 'macOS' && '0' || '' }} uses: devcontainers/ci@v0.2 with: subFolder: .github @@ -51,6 +53,8 @@ jobs: push: never - name: Test Container Image + env: + DOCKER_BUILDKIT: ${{ runner.os == 'macOS' && '0' || '' }} run: | docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" From 781f8ee62cbfb67ffc6130aeab41f650c132ea7a Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 02:13:18 +0700 Subject: [PATCH 23/57] add buildx plugin --- .github/workflows/devcontainer.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index d1ae2402a..ca1304bf8 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -31,9 +31,12 @@ jobs: - name: Setup docker (missing on MacOS) if: runner.os == 'macos' run: | - brew install docker + brew install docker docker-buildx brew install colima colima start + while ! docker info >/dev/null 2>&1; do sleep 1; done + mkdir -p ~/.docker/cli-plugins + ln -sfn $(which docker-buildx) ~/.docker/cli-plugins/docker-buildx - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -43,8 +46,6 @@ jobs: password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image - env: - DOCKER_BUILDKIT: ${{ runner.os == 'macOS' && '0' || '' }} uses: devcontainers/ci@v0.2 with: subFolder: .github @@ -53,8 +54,6 @@ jobs: push: never - name: Test Container Image - env: - DOCKER_BUILDKIT: ${{ runner.os == 'macOS' && '0' || '' }} run: | docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" From d6188d290f961f52efda6e4a699552240f66dde5 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Wed, 6 Nov 2024 06:29:17 +0100 Subject: [PATCH 24/57] include documentation for using docker container --- doc/docker_containers.rst | 52 ++++++++++++++++++++++++++++++++++----- doc/index.rst | 2 ++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index 556ef61ba..65436a8ff 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -1,7 +1,47 @@ -# Running PyPSA-Earth in Docker containers -## How to use Docker containers to run PyPSA-Earth. -- install Docker -- git clone the pypsa-earth repository -- open the pypsa-earth with VSCode -- rebuild and open the repo in a container \ No newline at end of file +.. _docker_containers: + +Alternate Installation with Docker +=============================================== + +This is an alternative way to create a development environment for PyPSA-Earth. This method is useful for users who are not familiar with programming or Python, or who do not want to install Python on their local machine. It uses Docker containers to create a development environment for PyPSA-Earth. + +This section provides a step-by-step guide on how to set up and use Docker containers to run PyPSA-Earth. + +Steps: + +1. Install Docker: Follow the instructions for your operating system: + - `Windows `_ + - `Linux `_ + - `MacOS `_ + + Ensure Docker is installed on your system. + +2. Install GitHub Desktop for your OS `here `_. + + + - Open GitHub Desktop. + - Click on "File" in the top left corner. + - Click on "Clone Repository". + - Paste the following URL in the URL field: + + .. code:: bash + + https://github.com/drifter089/pypsa-earth.git + + - Click on "Clone". + - Choose the location where you want to save the repository. + - Click on "Current Branch: main" and select `devContainers`. + - Click on "Open in Visual Studio Code". + + The repository will be cloned to your local machine. + +4. Rebuild and open in a container: + - Open the repository in VSCode. + - Click on the green icon in the bottom left corner of the VSCode window. + - Click on "Reopen in Container". + - Wait for the container to build and open the repository in the container. + + The environment will be ready for use. + + \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst index 0f7e5c20c..ca34d3271 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -128,6 +128,7 @@ Documentation * :doc:`introduction` * :doc:`installation` +* :doc:`docker_containers` * :doc:`short_tutorial` * :doc:`tutorial` * :doc:`data_workflow` @@ -140,6 +141,7 @@ Documentation introduction installation + docker_containers short_tutorial tutorial data_workflow From 49e5c5f3ec005459b4c0919688fa2a2e9d9dcec1 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Wed, 6 Nov 2024 06:30:31 +0100 Subject: [PATCH 25/57] update docker docs --- doc/docker_containers.rst | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index 65436a8ff..bc0ea71af 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -11,37 +11,38 @@ This section provides a step-by-step guide on how to set up and use Docker conta Steps: 1. Install Docker: Follow the instructions for your operating system: - - `Windows `_ - - `Linux `_ - - `MacOS `_ + +* `Windows `_ +* `Linux `_ +* `MacOS `_ Ensure Docker is installed on your system. 2. Install GitHub Desktop for your OS `here `_. +3. Clone the repository: + * Open GitHub Desktop. + * Click on "File" in the top left corner. + * Click on "Clone Repository". + * Paste the following URL in the URL field: - - Open GitHub Desktop. - - Click on "File" in the top left corner. - - Click on "Clone Repository". - - Paste the following URL in the URL field: - - .. code:: bash + .. code:: bash - https://github.com/drifter089/pypsa-earth.git + https://github.com/drifter089/pypsa-earth.git - - Click on "Clone". - - Choose the location where you want to save the repository. - - Click on "Current Branch: main" and select `devContainers`. - - Click on "Open in Visual Studio Code". + * Click on "Clone". + * Choose the location where you want to save the repository. + * Click on "Current Branch: main" and select `devContainers`. + * Click on "Open in Visual Studio Code". The repository will be cloned to your local machine. 4. Rebuild and open in a container: - - Open the repository in VSCode. - - Click on the green icon in the bottom left corner of the VSCode window. - - Click on "Reopen in Container". - - Wait for the container to build and open the repository in the container. + * Open the repository in VSCode. + * Click on the green icon in the bottom left corner of the VSCode window. + * Click on "Reopen in Container". + * Wait for the container to build and open the repository in the container. - The environment will be ready for use. + The environment will be ready for use. You can now run PyPSA-Earth in the container. \ No newline at end of file From 4117756d8e68174cfbe7b959e62dd6daeadc46da Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 7 Nov 2024 17:58:55 +0100 Subject: [PATCH 26/57] include docker readme --- Docker.MD | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 Docker.MD diff --git a/Docker.MD b/Docker.MD new file mode 100644 index 000000000..45e783e77 --- /dev/null +++ b/Docker.MD @@ -0,0 +1,49 @@ +# PyPSA-Earth Development Environment Setup with Docker + +This guide provides an alternative way to set up a development environment for **PyPSA-Earth** using Docker containers. This method is particularly useful for users who are not familiar with programming or Python, or who prefer not to install Python directly on their local machine. Using Docker simplifies setup by creating a self-contained environment for PyPSA-Earth. + +## Prerequisites + +### 1. Install Docker + +Ensure Docker is installed on your system. Follow the instructions for your operating system: + +- **Windows**: [Docker for Windows](https://docs.docker.com/desktop/install/windows-install/) +- **Linux**: [Docker for Linux](https://docs.docker.com/desktop/install/linux/) +- **MacOS**: [Docker for MacOS](https://docs.docker.com/desktop/install/mac-install/) + +### 2. Install GitHub Desktop + +You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install GitHub Desktop for your operating system from [here](https://desktop.github.com/download/). + +## Steps to Set Up PyPSA-Earth with Docker + +### Step 1: Clone the Repository + +1. Open **GitHub Desktop**. +2. Go to **File** > **Clone Repository**. +3. Paste the following URL in the URL field: + + ```bash + https://github.com/drifter089/pypsa-earth.git + ``` + +4. Click on **Clone**. +5. Choose the location where you want to save the repository on your local machine. +6. After cloning, click on **Current Branch: main** and select `devContainers`. +7. Click on **Open in Visual Studio Code**. + +### Step 2: Rebuild and Open in Container + +1. Open the cloned repository in **VSCode**. +2. Click on the green icon in the bottom left corner of the VSCode window. +3. Select **Reopen in Container**. +4. Wait for the container to build and for the repository to open in the container. + +Once these steps are completed, your development environment will be ready, and you can start using **PyPSA-Earth** in the Docker container. + +--- + +You are now all set up! You can use the development environment to explore PyPSA-Earth and make modifications as needed within the Docker container. + +You can start running the tutorial [here](https://pypsa-earth.readthedocs.io/en/latest/short_tutorial.html) \ No newline at end of file From 9a5dc0fcc4ba83eff8fd54e837dbd29abf8f0b6a Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 16:44:37 +0700 Subject: [PATCH 27/57] updated CI to build on ubuntu only and test on both --- .github/workflows/devcontainer.yml | 66 +++++++++++++++++++----------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index ca1304bf8..3eab92f3e 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -12,23 +12,40 @@ on: jobs: - build-and-push: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Build Dev Container Image + uses: devcontainers/ci@v0.2 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }} + push: never + + test: + needs: build strategy: fail-fast: false matrix: - os: - - [ubuntu-latest] - - [ macos-13 ] + os: [ubuntu-latest, macos-13] runs-on: ${{ matrix.os }} steps: - - - name: Checkout - id: checkout - uses: actions/checkout@v1 - - - name: Setup docker (missing on MacOS) + - name: Setup docker (macOS only) if: runner.os == 'macos' run: | brew install docker docker-buildx @@ -37,27 +54,26 @@ jobs: while ! docker info >/dev/null 2>&1; do sleep 1; done mkdir -p ~/.docker/cli-plugins ln -sfn $(which docker-buildx) ~/.docker/cli-plugins/docker-buildx - + + - name: Test Container Image + run: | + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" + + push: + needs: [build, test] + runs-on: ubuntu-latest + if: ${{ always() && success() }} + + steps: - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Build Dev Container Image - uses: devcontainers/ci@v0.2 - with: - subFolder: .github - imageName: ghcr.io/${{ github.repository }} - cacheFrom: ghcr.io/${{ github.repository }} - push: never - - - name: Test Container Image - run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - name: Push Container Image - if: ${{ success() }} run: | - docker push ghcr.io/${{ github.repository }} \ No newline at end of file + docker push ghcr.io/${{ github.repository }} + + From 07590df296762487364b05cdcdb4408e33a00856 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:12:14 +0700 Subject: [PATCH 28/57] pushing test image before runningn tests --- .github/workflows/devcontainer.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 3eab92f3e..6f3030b3d 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -31,9 +31,9 @@ jobs: uses: devcontainers/ci@v0.2 with: subFolder: .github - imageName: ghcr.io/${{ github.repository }} - cacheFrom: ghcr.io/${{ github.repository }} - push: never + imageName: ghcr.io/${{ github.repository }}:test + cacheFrom: ghcr.io/${{ github.repository }}:test + push: always test: needs: build @@ -57,7 +57,7 @@ jobs: - name: Test Container Image run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" + docker run --rm ghcr.io/${{ github.repository }}:test /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" push: needs: [build, test] @@ -74,6 +74,6 @@ jobs: - name: Push Container Image run: | - docker push ghcr.io/${{ github.repository }} + docker push ghcr.io/${{ github.repository }}:latest From 9629a2e5d577b8b33ac6377cd0e0046350a76829 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:21:00 +0700 Subject: [PATCH 29/57] adding updated build action --- .github/workflows/devcontainer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 6f3030b3d..ce80cf933 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -28,7 +28,7 @@ jobs: password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image - uses: devcontainers/ci@v0.2 + uses: devcontainers/ci@v0.3 with: subFolder: .github imageName: ghcr.io/${{ github.repository }}:test From fc27e5a7ae43b8677a04361a0a3ab4f9db68d6d6 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:28:15 +0700 Subject: [PATCH 30/57] removing cache --- .github/workflows/devcontainer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index ce80cf933..416661803 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -32,7 +32,6 @@ jobs: with: subFolder: .github imageName: ghcr.io/${{ github.repository }}:test - cacheFrom: ghcr.io/${{ github.repository }}:test push: always test: From a0ba73a84bee6416c5ddf34405350589d0497957 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:30:48 +0700 Subject: [PATCH 31/57] removing tagging --- .github/workflows/devcontainer.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 416661803..1f03a34d0 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -31,7 +31,8 @@ jobs: uses: devcontainers/ci@v0.3 with: subFolder: .github - imageName: ghcr.io/${{ github.repository }}:test + cacheFrom: ghcr.io/${{ github.repository }} + imageName: ghcr.io/${{ github.repository }} push: always test: @@ -56,7 +57,7 @@ jobs: - name: Test Container Image run: | - docker run --rm ghcr.io/${{ github.repository }}:test /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" push: needs: [build, test] @@ -73,6 +74,6 @@ jobs: - name: Push Container Image run: | - docker push ghcr.io/${{ github.repository }}:latest + docker push ghcr.io/${{ github.repository }} From e1c40540813f0c799ed8327862a005a416f26e5a Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:39:58 +0700 Subject: [PATCH 32/57] adding no cache to build --- .github/workflows/devcontainer.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 1f03a34d0..65d2fff23 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -34,13 +34,14 @@ jobs: cacheFrom: ghcr.io/${{ github.repository }} imageName: ghcr.io/${{ github.repository }} push: always + noCache: true test: needs: build strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-13] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} From bc4e8fbd5cb171d9e89a97f14cd87303ac335016 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:44:13 +0700 Subject: [PATCH 33/57] no need to push again after checking --- .github/workflows/devcontainer.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 65d2fff23..8a3cb8372 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -60,21 +60,4 @@ jobs: run: | docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - push: - needs: [build, test] - runs-on: ubuntu-latest - if: ${{ always() && success() }} - - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Push Container Image - run: | - docker push ghcr.io/${{ github.repository }} - From 4d98000cf510aaf1dedfbd39e004478b978f5e39 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 6 Dec 2024 13:07:59 +0100 Subject: [PATCH 34/57] Update devcontainer.yml --- .github/workflows/devcontainer.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 8a3cb8372..4d1e86528 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -35,29 +35,3 @@ jobs: imageName: ghcr.io/${{ github.repository }} push: always noCache: true - - test: - needs: build - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - - runs-on: ${{ matrix.os }} - - steps: - - name: Setup docker (macOS only) - if: runner.os == 'macos' - run: | - brew install docker docker-buildx - brew install colima - colima start - while ! docker info >/dev/null 2>&1; do sleep 1; done - mkdir -p ~/.docker/cli-plugins - ln -sfn $(which docker-buildx) ~/.docker/cli-plugins/docker-buildx - - - name: Test Container Image - run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - - From c610ddc8d875c4c5741723a6ae22dc0e0a090c65 Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Fri, 6 Dec 2024 19:14:24 +0700 Subject: [PATCH 35/57] Update Dockerfile using linux-pinned --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2b171658..57e72d6a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /pypsa-earth COPY ./envs ./temp -RUN conda env create -n pypsa-earth -f temp/environment.yaml +RUN conda env create -n pypsa-earth -f temp/linux-pinned.yaml RUN conda init bash @@ -27,7 +27,5 @@ RUN rm -r temp RUN conda clean -afy && \ rm -rf /tmp/* -ENV LD_PRELOAD=/opt/conda/envs/pypsa-earth/lib/python3.10/site-packages/sklearn/utils/../../../../libgomp.so.1 - CMD ["bash"] From 9c983caeb330a79c6692da2b97ed3b02a9f19f11 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 6 Dec 2024 13:26:56 +0100 Subject: [PATCH 36/57] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 57e72d6a4..a480b3780 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ RUN conda update -n base conda RUN conda install -n base conda-libmamba-solver RUN conda config --set solver libmamba -RUN apt-get update && apt-get install -y bash git +RUN apt-get update && apt-get install -y bash git && apt-get install gcc -y WORKDIR /pypsa-earth From 68bb45d0fc23bfd1c3ac162ffb8547035e90ea13 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 11 Dec 2024 16:59:29 +0700 Subject: [PATCH 37/57] pre-commit bot run --- .devcontainer/devcontainer.json | 2 +- .devcontainer/setup.sh | 4 ++-- .devcontainer/welcome-message.txt | 2 +- .github/.devcontainer/test.sh | 2 +- .github/workflows/devcontainer.yml | 10 +++++----- Docker.MD | 2 +- Dockerfile | 1 - doc/docker_containers.rst | 2 +- 8 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 81e1605d9..fe8c094b5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ -// For format details, see https://aka.ms/devcontainer.json. +// For format details, see https://aka.ms/devcontainer.json. { "name": "pypsa earth dev", "image": "ghcr.io/drifter089/pypsa-earth:latest", diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index e2fc1a3e8..deec01786 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,6 +1,6 @@ #!/bin/sh -# mv ../pypsa-eur/* ./ +# mv ../pypsa-eur/* ./ # echo 'solved tutorial files are copied to workspace' -cat .devcontainer/welcome-message.txt \ No newline at end of file +cat .devcontainer/welcome-message.txt diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt index 9fe33b7d6..081694bdf 100644 --- a/.devcontainer/welcome-message.txt +++ b/.devcontainer/welcome-message.txt @@ -18,4 +18,4 @@ We’re excited to have you here! This setup allows you to contribute to PyPSA-E 🚀 Start Exploring and Happy Coding! -Don’t hesitate to reach out if you need help—our community is here to support you. \ No newline at end of file +Don’t hesitate to reach out if you need help—our community is here to support you. diff --git a/.github/.devcontainer/test.sh b/.github/.devcontainer/test.sh index c4ac3f290..78556dbcd 100644 --- a/.github/.devcontainer/test.sh +++ b/.github/.devcontainer/test.sh @@ -2,4 +2,4 @@ cp config.tutorial.yaml config.yaml -snakemake -j 1 solve_all_networks \ No newline at end of file +snakemake -j 1 solve_all_networks diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 4d1e86528..222048c33 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -4,13 +4,13 @@ on: workflow_dispatch: push: branches: - - "main" + - "main" tags: - - "v*.*.*" + - "v*.*.*" pull_request: - branches: [main] + branches: [main] + - jobs: build: runs-on: ubuntu-latest @@ -33,5 +33,5 @@ jobs: subFolder: .github cacheFrom: ghcr.io/${{ github.repository }} imageName: ghcr.io/${{ github.repository }} - push: always + push: always noCache: true diff --git a/Docker.MD b/Docker.MD index 45e783e77..e7adbc01c 100644 --- a/Docker.MD +++ b/Docker.MD @@ -46,4 +46,4 @@ Once these steps are completed, your development environment will be ready, and You are now all set up! You can use the development environment to explore PyPSA-Earth and make modifications as needed within the Docker container. -You can start running the tutorial [here](https://pypsa-earth.readthedocs.io/en/latest/short_tutorial.html) \ No newline at end of file +You can start running the tutorial [here](https://pypsa-earth.readthedocs.io/en/latest/short_tutorial.html) diff --git a/Dockerfile b/Dockerfile index a480b3780..bb831e973 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,3 @@ RUN conda clean -afy && \ rm -rf /tmp/* CMD ["bash"] - diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index bc0ea71af..fdc561f83 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -45,4 +45,4 @@ Steps: The environment will be ready for use. You can now run PyPSA-Earth in the container. - \ No newline at end of file + From dca591b2cdfd4b3f18eebdb192730d708f64c6d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:03:11 +0000 Subject: [PATCH 38/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/docker_containers.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index fdc561f83..df979711b 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -44,5 +44,3 @@ Steps: * Wait for the container to build and open the repository in the container. The environment will be ready for use. You can now run PyPSA-Earth in the container. - - From 6effd98172eafdd8f4d7d03b3d8e1c97321d2e0f Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:30:36 +0700 Subject: [PATCH 39/57] Delete .github/.devcontainer/test.sh --- .github/.devcontainer/test.sh | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .github/.devcontainer/test.sh diff --git a/.github/.devcontainer/test.sh b/.github/.devcontainer/test.sh deleted file mode 100644 index 78556dbcd..000000000 --- a/.github/.devcontainer/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cp config.tutorial.yaml config.yaml - -snakemake -j 1 solve_all_networks From d793625e5afa112c526087062c1e5f924a8405ab Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:31:37 +0700 Subject: [PATCH 40/57] Update devcontainer.yml --- .github/workflows/devcontainer.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 222048c33..5148df2be 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -3,13 +3,8 @@ name: Dev Container Build and Push Image on: workflow_dispatch: push: - branches: - - "main" - tags: - - "v*.*.*" - pull_request: - branches: [main] - + paths: + - envs/linux-pinned.yaml jobs: build: From 4d659a3299ea20c572738445a18119040da0f89a Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 13 Dec 2024 13:08:09 +0100 Subject: [PATCH 41/57] fix precommit issues --- .devcontainer/devcontainer.json | 3 +++ .devcontainer/setup.sh | 3 +++ .devcontainer/welcome-message.txt | 4 +++- Docker.MD | 5 +++++ Dockerfile | 3 +++ doc/docker_containers.rst | 3 +++ 6 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fe8c094b5..02a0c928d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +// +// SPDX-License-Identifier: CC-BY-4.0 // For format details, see https://aka.ms/devcontainer.json. { "name": "pypsa earth dev", diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index deec01786..f57177eea 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 #!/bin/sh # mv ../pypsa-eur/* ./ diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt index 081694bdf..afcec3035 100644 --- a/.devcontainer/welcome-message.txt +++ b/.devcontainer/welcome-message.txt @@ -1,4 +1,6 @@ - +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 diff --git a/Docker.MD b/Docker.MD index e7adbc01c..c2db2aaa7 100644 --- a/Docker.MD +++ b/Docker.MD @@ -1,3 +1,8 @@ + # PyPSA-Earth Development Environment Setup with Docker This guide provides an alternative way to set up a development environment for **PyPSA-Earth** using Docker containers. This method is particularly useful for users who are not familiar with programming or Python, or who prefer not to install Python directly on their local machine. Using Docker simplifies setup by creating a self-contained environment for PyPSA-Earth. diff --git a/Dockerfile b/Dockerfile index bb831e973..1873210a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 FROM condaforge/mambaforge RUN conda update -n base conda diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index df979711b..bc55423b7 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -1,3 +1,6 @@ +.. SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +.. +.. SPDX-License-Identifier: CC-BY-4.0 .. _docker_containers: From 1420450403b6101282960e39672d1b5edbca378c Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Sat, 14 Dec 2024 19:58:22 +0700 Subject: [PATCH 42/57] trigger and image name --- .github/workflows/devcontainer.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 5148df2be..12288a6b1 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -3,6 +3,7 @@ name: Dev Container Build and Push Image on: workflow_dispatch: push: + branches: [main] paths: - envs/linux-pinned.yaml @@ -26,7 +27,7 @@ jobs: uses: devcontainers/ci@v0.3 with: subFolder: .github - cacheFrom: ghcr.io/${{ github.repository }} - imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }}-dev-env + imageName: ghcr.io/${{ github.repository }}-dev-env push: always - noCache: true + # noCache: true From c8654cfeaf146c90b96961e31bfd41f9f235bb79 Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Sat, 14 Dec 2024 20:02:16 +0700 Subject: [PATCH 43/57] Update setup.sh --- .devcontainer/setup.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index f57177eea..a3ef9d7bb 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -3,7 +3,4 @@ # SPDX-License-Identifier: CC-BY-4.0 #!/bin/sh -# mv ../pypsa-eur/* ./ -# echo 'solved tutorial files are copied to workspace' - cat .devcontainer/welcome-message.txt From e03d724948c8ccadebd8583add5e9ae5614b9e03 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Sat, 21 Dec 2024 13:40:06 +0100 Subject: [PATCH 44/57] restore BJ to country selection --- config.tutorial.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 3ad688678..1c5bf7552 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -6,7 +6,7 @@ version: 0.4.2 tutorial: true -countries: ["NG"] +countries: ["NG", "BJ"] enable: retrieve_databundle: true From 62a4bfaddcab2010e2c187bcfb905ada8b869f3b Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 17:50:44 +0100 Subject: [PATCH 45/57] update FT to account for Hydrogen and Electricity input (#1226) * update FT to account for Hydrogen and electricity input * revert implementation * include electricity bus * added electricity bus * revert snakemock * Update release_notes.rst * revert efficiency for FT --- doc/release_notes.rst | 1 + scripts/prepare_sector_network.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0156ec70e..0f29269f9 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -18,6 +18,7 @@ This part of documentation collects descriptive release notes to capture the mai **Minor Changes and bug-fixing** +* Added electricity bus to Fischer-Tropsch in prepare_sector_network.py `PR #1226 `__ PyPSA-Earth 0.5.0 diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 796125b95..880dafef7 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -170,6 +170,7 @@ def H2_liquid_fossil_conversions(n, costs): bus0=spatial.nodes + " H2", bus1=spatial.oil.nodes, bus2=spatial.co2.nodes, + bus3=spatial.nodes, carrier="Fischer-Tropsch", efficiency=costs.at["Fischer-Tropsch", "efficiency"], capital_cost=costs.at["Fischer-Tropsch", "fixed"] @@ -178,6 +179,8 @@ def H2_liquid_fossil_conversions(n, costs): ], # Use efficiency to convert from EUR/MW_FT/a to EUR/MW_H2/a efficiency2=-costs.at["oil", "CO2 intensity"] * costs.at["Fischer-Tropsch", "efficiency"], + efficiency3=-costs.at["Fischer-Tropsch", "electricity-input"] + / costs.at["Fischer-Tropsch", "hydrogen-input"], p_nom_extendable=True, p_min_pu=options.get("min_part_load_fischer_tropsch", 0), lifetime=costs.at["Fischer-Tropsch", "lifetime"], From 236e0d0064bd3fdb2d3577274e13057da3233a9a Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 23 Dec 2024 18:15:17 +0100 Subject: [PATCH 46/57] Bypass external urls (#1257) * Update bundles with gadm-like to bypass gadm dependency * Minor fixes * Restore automatic emission download * Add release note * Add description to bundles --- configs/bundle_config.yaml | 43 ++++++++++++++++++++++++++++++++++---- doc/release_notes.rst | 2 ++ scripts/prepare_network.py | 5 +++-- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 7d1f55103..2f2824a54 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -36,11 +36,15 @@ databundles: category: data destination: "data" urls: - zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_NGBJ.zip?download=1 - gdrive: https://drive.google.com/file/d/1Vb1ISjhy7iwTTZYeezGd6S4nLt-EDGme/view?usp=drive_link + zenodo: https://sandbox.zenodo.org/records/145504/files/bundle_tutorial_NGBJ_with_gadmlike.zip?download=1 + gdrive: https://drive.google.com/file/d/12K03Epx3O9o-IQLh9afzCQyT-nMKWM3P/view?usp=drive_link output: - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + - data/gadm/gadm41_NGA/gadm41_NGA.gpkg # needed in build_shapes + - data/gadm/gadm41_BEN/gadm41_BEN.gpkg # needed in build_shapes + - data/gadm/gadm36_NGA/gadm36_NGA.gpkg # needed in sector-coupled model + - data/gadm/gadm36_BEN/gadm36_BEN.gpkg # needed in sector-coupled model # tutorial bundle specific for Botswana only bundle_tutorial_BW: @@ -49,11 +53,13 @@ databundles: category: data destination: "data" urls: - zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_BW.zip?download=1 - gdrive: https://drive.google.com/file/d/19IXvTD8gVSzgTInL85ta7QjaNI8ZPCCY/view?usp=drive_link + zenodo: https://sandbox.zenodo.org/records/145504/files/bundle_tutorial_BW_with_gadmlike.zip?download=1 + gdrive: https://drive.google.com/file/d/1YbbYGs1NsSsZYqNX1g1Jo-iJzt5m-81c/view?usp=drive_link output: - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + - data/gadm/gadm41_BWA/gadm41_BWA.gpkg # needed in build_shapes + - data/gadm/gadm36_BWA/gadm36_BWA.gpkg # needed in sector-coupled model # tutorial bundle specific for Morocco only bundle_tutorial_MA: @@ -364,3 +370,32 @@ databundles: urls: protectedplanet: https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{month:s}{year:d}_Public_shp.zip output: [data/landcover/world_protected_areas/*] + + # Backup tutorial bundles with no gadm-like data; for reference: + # https://github.com/pypsa-meets-earth/pypsa-earth/issues/1258 + # + # # tutorial bundle specific for Nigeria and Benin only, without gadm-like data + # bundle_tutorial_NGBJ: + # countries: [NG, BJ] + # tutorial: true + # category: data + # destination: "data" + # urls: + # zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_NGBJ.zip?download=1 + # gdrive: https://drive.google.com/file/d/1Vb1ISjhy7iwTTZYeezGd6S4nLt-EDGme/view?usp=drive_link + # output: + # - data/gebco/GEBCO_2021_TID.nc + # - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + + # # tutorial bundle specific for Botswana only, without gadm-like data + # bundle_tutorial_BW: + # countries: [BW] + # tutorial: true + # category: data + # destination: "data" + # urls: + # zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_BW.zip?download=1 + # gdrive: https://drive.google.com/file/d/19IXvTD8gVSzgTInL85ta7QjaNI8ZPCCY/view?usp=drive_link + # output: + # - data/gebco/GEBCO_2021_TID.nc + # - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0f29269f9..bd61a857e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -20,6 +20,8 @@ This part of documentation collects descriptive release notes to capture the mai * Added electricity bus to Fischer-Tropsch in prepare_sector_network.py `PR #1226 `__ +* Update BW, NG and BJ tutorial databundles to include gadm-like sources from geoboundaries `PR #1257 `__ + PyPSA-Earth 0.5.0 ================= diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 3d6c73cb8..dc03380db 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -325,6 +325,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): clusters="4", ll="c1", opts="Co2L-4H", + configfile="test/config.sector.yaml", ) configure_logging(snakemake) @@ -372,10 +373,10 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): co2limit = co2limit * float(m[0]) logger.info("Setting CO2 limit according to emission base year.") elif len(m) > 0: - co2limit = float(m[0]) * snakemake.params.electricity["co2base"] + co2limit = float(m[0]) * float(snakemake.params.electricity["co2base"]) logger.info("Setting CO2 limit according to wildcard value.") else: - co2limit = snakemake.params.electricity["co2limit"] + co2limit = float(snakemake.params.electricity["co2limit"]) logger.info("Setting CO2 limit according to config value.") add_co2limit(n, co2limit, Nyears) break From 3dc6ddcd0421581ca31c6ef0016530e187dece41 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 23 Dec 2024 18:35:56 +0100 Subject: [PATCH 47/57] update gdrive link (#1259) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15556eab2..34b998185 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ by [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pypsa-meets-earth/pypsa-earth/main.svg)](https://results.pre-commit.ci/latest/github/pypsa-meets-earth/pypsa-earth/main) [![Discord](https://img.shields.io/discord/911692131440148490?logo=discord)](https://discord.gg/AnuJBk23FU) -[![Google Drive](https://img.shields.io/badge/Google%20Drive-4285F4?style=flat&logo=googledrive&logoColor=white)](https://drive.google.com/drive/folders/1U7fgktbxlaGzWxT2C0-Xv-_ffWCxAKZz) +[![Google Drive](https://img.shields.io/badge/Google%20Drive-4285F4?style=flat&logo=googledrive&logoColor=white)](https://drive.google.com/drive/folders/13Z8Y9zgsh5IZaDNkkRyo1wkoMgbdUxT5?usp=sharing) **PyPSA-Earth: A Global Sector-Coupled Open-Source Multi-Energy System Model** From a0fa30f2c1179797b3ad50913f8380cb7ec008f2 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 23 Dec 2024 19:23:48 +0100 Subject: [PATCH 48/57] Fix docs (#1260) * Update env requirements for doc * Update documentation environment --- doc/requirements.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index d2b518fd8..1dd52eaaf 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -8,7 +8,7 @@ sphinx_book_theme sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 -pypsa +pypsa >=0.24, <0.25 vresutils>=0.3.1 powerplantmatching>=0.5.5 atlite>=0.2.9 @@ -33,7 +33,9 @@ gitpython chaospy numba ruamel.yaml<=0.17.26 -earth-osm>=0.1.0, <0.2.0 +earth-osm>=2.3.post1 reverse-geocode pyDOE2 -# graphviz +graphviz + +fake_useragent From afd97122d4db7946406aa771455903d1bf585fb0 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:24:41 +0100 Subject: [PATCH 49/57] Include dev container and docker feature --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index bd61a857e..03b08a61e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -15,6 +15,7 @@ This part of documentation collects descriptive release notes to capture the mai * Include option in the config to allow for custom airport data `PR #1241 `__ +* Added Dev Containers and docker as an option to get started with pypsa-earth `PR #1228 `__ **Minor Changes and bug-fixing** From 5c5c2b0138cc7d526736b571b48b174eac78b1c1 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Mon, 23 Dec 2024 20:29:07 +0100 Subject: [PATCH 50/57] Restore environment (#1048) * Remove a constraint on geopandas * Restore earth-osm conda --------- Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Co-authored-by: Davide Fioriti --- envs/environment.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index b82cf3aa2..96278f6f2 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -17,9 +17,7 @@ dependencies: # currently the packages are being installed with pip # need to move back to conda once the issues will be resolved - powerplantmatching>=0.5.19 -# - earth-osm>=2.1 -# until the release will incorporate all the fixes needed -# to work with CDS beta +- earth-osm>=2.3.post1 - atlite>=0.3 # Dependencies of the workflow itself @@ -35,8 +33,8 @@ dependencies: # starting from 1.3.5 numpoly requires numpy>2.0 which leads to issues - numpoly<=1.3.4 - pandas -- geopandas>=0.11.0, <=0.14.3 -- fiona<1.10.0 +- geopandas>=0.11.0 +- fiona!=1.8.22 - xarray>=2023.11.0, <2023.12.0 - netcdf4 - networkx @@ -86,7 +84,6 @@ dependencies: - gurobi - pip: - - earth-osm==2.2 # until conda release it out for earth-osm - git+https://github.com/davide-f/google-drive-downloader@master # google drive with fix for virus scan - tsam>=1.1.0 - chaospy # lastest version only available on pip From 4760a3966629efccd33284c18fabbcbfbdb758d1 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:36:43 +0100 Subject: [PATCH 51/57] Update welcome message --- .devcontainer/welcome-message.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt index afcec3035..709322166 100644 --- a/.devcontainer/welcome-message.txt +++ b/.devcontainer/welcome-message.txt @@ -6,11 +6,11 @@ 👋 Welcome to the PyPSA-Earth Development Environment! -We’re excited to have you here! This setup allows you to contribute to PyPSA-Earth effortlessly using a development container in VS Code. +We’re excited to have you here! This setup allows you to use and contribute to PyPSA-Earth using a development container in VS Code. 📖 Getting Started for New Users - • For a step-by-step guide on setting up your environment, debugging, and making your first contribution, refer to the PyPSA-Earth README. It covers everything you need to know as a newcomer. + • For a step-by-step guide on setting up your environment, debugging, and making your first contribution, refer to the PyPSA-Earth README here: (https://github.com/pypsa-meets-earth/pypsa-earth/blob/main/README.md). It covers everything you need to know as a newcomer. • The configuration files for the development container are located in the .github/.devcontainer folder. 💡 Tips for New Users @@ -20,4 +20,4 @@ We’re excited to have you here! This setup allows you to contribute to PyPSA-E 🚀 Start Exploring and Happy Coding! -Don’t hesitate to reach out if you need help—our community is here to support you. +Don’t hesitate to reach out if you need help—our community is here to support you. You can access our discord server here: https://discord.gg/AnuJBk23FU From c931b513e00c7f013da0908a12201277b2d0e3e6 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:41:06 +0100 Subject: [PATCH 52/57] Update devcontainer.yml --- .github/workflows/devcontainer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 12288a6b1..85de566fe 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -30,4 +30,3 @@ jobs: cacheFrom: ghcr.io/${{ github.repository }}-dev-env imageName: ghcr.io/${{ github.repository }}-dev-env push: always - # noCache: true From 1c3e3f4a6ef598e12dcbf582628fcea0477df2d0 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:45:18 +0100 Subject: [PATCH 53/57] Update Docker.MD --- Docker.MD | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Docker.MD b/Docker.MD index c2db2aaa7..75e99f0cc 100644 --- a/Docker.MD +++ b/Docker.MD @@ -17,7 +17,11 @@ Ensure Docker is installed on your system. Follow the instructions for your oper - **Linux**: [Docker for Linux](https://docs.docker.com/desktop/install/linux/) - **MacOS**: [Docker for MacOS](https://docs.docker.com/desktop/install/mac-install/) -### 2. Install GitHub Desktop +### 2. Install Visual Studio Code(VSC) + +You can use the link [here](https://code.visualstudio.com/download) to install Visual Studio Code on your operating system. Ensure to select the most compatible file for your operating system. + +### 3. Install GitHub Desktop You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install GitHub Desktop for your operating system from [here](https://desktop.github.com/download/). From 65c1ca3e3c85cb4c370769b4af0b451e77bf6414 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:48:55 +0100 Subject: [PATCH 54/57] Update Docker.MD --- Docker.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker.MD b/Docker.MD index 75e99f0cc..16e96e0dc 100644 --- a/Docker.MD +++ b/Docker.MD @@ -45,7 +45,7 @@ You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install G ### Step 2: Rebuild and Open in Container 1. Open the cloned repository in **VSCode**. -2. Click on the green icon in the bottom left corner of the VSCode window. +2. Click on the icon located at the bottom left corner of the VSCode window. 3. Select **Reopen in Container**. 4. Wait for the container to build and for the repository to open in the container. From 310635648f739b11d547fc2cb11a5d2dea0365a7 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:49:48 +0100 Subject: [PATCH 55/57] Update Docker.MD --- Docker.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker.MD b/Docker.MD index 16e96e0dc..24e8a9d84 100644 --- a/Docker.MD +++ b/Docker.MD @@ -34,7 +34,7 @@ You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install G 3. Paste the following URL in the URL field: ```bash - https://github.com/drifter089/pypsa-earth.git + https://github.com/pypsa-meets-earth/pypsa-earth.git ``` 4. Click on **Clone**. From 5c96022de4f26bcc7e22d47c03373049daf50d4a Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:53:34 +0100 Subject: [PATCH 56/57] revert config.tutorial.yaml --- config.tutorial.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 9c2b87d1a..7ada63032 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,7 +9,6 @@ tutorial: true countries: ["NG", "BJ"] enable: - retrieve_databundle: true build_natura_raster: true progress_bar: false From 0c042a0a4268fe0ac463c24c25d17d8416a69a54 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:57:24 +0100 Subject: [PATCH 57/57] Update docker_containers.rst --- doc/docker_containers.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index bc55423b7..68a944cb9 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -21,9 +21,11 @@ Steps: Ensure Docker is installed on your system. -2. Install GitHub Desktop for your OS `here `_. +2. You can use the link `here `_ to install Visual Studio Code on your operating system. Ensure to select the most compatible file for your operating system. -3. Clone the repository: +3. Install GitHub Desktop for your OS `here `_. + +4. Clone the repository: * Open GitHub Desktop. * Click on "File" in the top left corner. * Click on "Clone Repository". @@ -31,7 +33,7 @@ Steps: .. code:: bash - https://github.com/drifter089/pypsa-earth.git + https://github.com/pypsa-meets-earth/pypsa-earth.git * Click on "Clone". * Choose the location where you want to save the repository. @@ -40,9 +42,9 @@ Steps: The repository will be cloned to your local machine. -4. Rebuild and open in a container: +5. Rebuild and open in a container: * Open the repository in VSCode. - * Click on the green icon in the bottom left corner of the VSCode window. + * Click on the icon in the far bottom left corner of the VSCode window. * Click on "Reopen in Container". * Wait for the container to build and open the repository in the container.