diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..02a0c928d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +// +// SPDX-License-Identifier: CC-BY-4.0 +// For format details, see https://aka.ms/devcontainer.json. +{ + "name": "pypsa earth dev", + "image": "ghcr.io/drifter089/pypsa-earth:latest", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces,type=bind,consistency=cached", + "initializeCommand": "docker pull ghcr.io/drifter089/pypsa-earth:latest", + "workspaceFolder": "/workspaces", + "postAttachCommand": "bash .devcontainer/setup.sh" +} diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100644 index 000000000..a3ef9d7bb --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 +#!/bin/sh + +cat .devcontainer/welcome-message.txt diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt new file mode 100644 index 000000000..709322166 --- /dev/null +++ b/.devcontainer/welcome-message.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 + + + +👋 Welcome to the PyPSA-Earth Development Environment! + +We’re excited to have you here! This setup allows you to use and contribute to PyPSA-Earth using a development container in VS Code. + +📖 Getting Started for New Users + + • For a step-by-step guide on setting up your environment, debugging, and making your first contribution, refer to the PyPSA-Earth README here: (https://github.com/pypsa-meets-earth/pypsa-earth/blob/main/README.md). It covers everything you need to know as a newcomer. + • The configuration files for the development container are located in the .github/.devcontainer folder. + +💡 Tips for New Users + + • Make the most of VS Code by using the Command Palette (Cmd/Ctrl + Shift + P or F1) for quick access to features and commands. + • If you’re new to development containers, learn the basics at containers.dev. + +🚀 Start Exploring and Happy Coding! + +Don’t hesitate to reach out if you need help—our community is here to support you. You can access our discord server here: https://discord.gg/AnuJBk23FU diff --git a/.github/.devcontainer/devcontainer.json b/.github/.devcontainer/devcontainer.json new file mode 100644 index 000000000..fda35213c --- /dev/null +++ b/.github/.devcontainer/devcontainer.json @@ -0,0 +1,33 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu +{ + "name": "pypsa earth dev", + "build": { + "dockerfile": "../../Dockerfile" + }, + + "features": { + "ghcr.io/devcontainers-contrib/features/bash-command:1": {}, + "ghcr.io/eliises/devcontainer-features/bash-profile:1": {} + }, + "customizations": { + "vscode": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/bash" + } + }, + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-azuretools.vscode-docker", + "ms-toolsai.jupyter", + "zainchen.json", + "tomoki1207.pdf", + "grapecity.gc-excelviewer" + ] + } + }, + + "postCreateCommand": "python -m pip install --upgrade debugpy" +} diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml new file mode 100644 index 000000000..85de566fe --- /dev/null +++ b/.github/workflows/devcontainer.yml @@ -0,0 +1,32 @@ +name: Dev Container Build and Push Image + +on: + workflow_dispatch: + push: + branches: [main] + paths: + - envs/linux-pinned.yaml + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Build Dev Container Image + uses: devcontainers/ci@v0.3 + with: + subFolder: .github + cacheFrom: ghcr.io/${{ github.repository }}-dev-env + imageName: ghcr.io/${{ github.repository }}-dev-env + push: always diff --git a/Docker.MD b/Docker.MD new file mode 100644 index 000000000..24e8a9d84 --- /dev/null +++ b/Docker.MD @@ -0,0 +1,58 @@ + +# PyPSA-Earth Development Environment Setup with Docker + +This guide provides an alternative way to set up a development environment for **PyPSA-Earth** using Docker containers. This method is particularly useful for users who are not familiar with programming or Python, or who prefer not to install Python directly on their local machine. Using Docker simplifies setup by creating a self-contained environment for PyPSA-Earth. + +## Prerequisites + +### 1. Install Docker + +Ensure Docker is installed on your system. Follow the instructions for your operating system: + +- **Windows**: [Docker for Windows](https://docs.docker.com/desktop/install/windows-install/) +- **Linux**: [Docker for Linux](https://docs.docker.com/desktop/install/linux/) +- **MacOS**: [Docker for MacOS](https://docs.docker.com/desktop/install/mac-install/) + +### 2. Install Visual Studio Code(VSC) + +You can use the link [here](https://code.visualstudio.com/download) to install Visual Studio Code on your operating system. Ensure to select the most compatible file for your operating system. + +### 3. Install GitHub Desktop + +You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install GitHub Desktop for your operating system from [here](https://desktop.github.com/download/). + +## Steps to Set Up PyPSA-Earth with Docker + +### Step 1: Clone the Repository + +1. Open **GitHub Desktop**. +2. Go to **File** > **Clone Repository**. +3. Paste the following URL in the URL field: + + ```bash + https://github.com/pypsa-meets-earth/pypsa-earth.git + ``` + +4. Click on **Clone**. +5. Choose the location where you want to save the repository on your local machine. +6. After cloning, click on **Current Branch: main** and select `devContainers`. +7. Click on **Open in Visual Studio Code**. + +### Step 2: Rebuild and Open in Container + +1. Open the cloned repository in **VSCode**. +2. Click on the icon located at the bottom left corner of the VSCode window. +3. Select **Reopen in Container**. +4. Wait for the container to build and for the repository to open in the container. + +Once these steps are completed, your development environment will be ready, and you can start using **PyPSA-Earth** in the Docker container. + +--- + +You are now all set up! You can use the development environment to explore PyPSA-Earth and make modifications as needed within the Docker container. + +You can start running the tutorial [here](https://pypsa-earth.readthedocs.io/en/latest/short_tutorial.html) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..1873210a0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 +FROM condaforge/mambaforge + +RUN conda update -n base conda +RUN conda install -n base conda-libmamba-solver +RUN conda config --set solver libmamba + +RUN apt-get update && apt-get install -y bash git && apt-get install gcc -y + +WORKDIR /pypsa-earth + +COPY ./envs ./temp + +RUN conda env create -n pypsa-earth -f temp/linux-pinned.yaml + +RUN conda init bash + +RUN touch ~/.bashrc && echo "conda activate pypsa-earth" >> ~/.bashrc + +SHELL ["/bin/bash", "--login", "-c"] + +ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH + +RUN conda install conda-forge::openjdk -y + +RUN rm -r temp + +RUN conda clean -afy && \ + rm -rf /tmp/* + +CMD ["bash"] diff --git a/README.md b/README.md index 15556eab2..34b998185 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ by [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pypsa-meets-earth/pypsa-earth/main.svg)](https://results.pre-commit.ci/latest/github/pypsa-meets-earth/pypsa-earth/main) [![Discord](https://img.shields.io/discord/911692131440148490?logo=discord)](https://discord.gg/AnuJBk23FU) -[![Google Drive](https://img.shields.io/badge/Google%20Drive-4285F4?style=flat&logo=googledrive&logoColor=white)](https://drive.google.com/drive/folders/1U7fgktbxlaGzWxT2C0-Xv-_ffWCxAKZz) +[![Google Drive](https://img.shields.io/badge/Google%20Drive-4285F4?style=flat&logo=googledrive&logoColor=white)](https://drive.google.com/drive/folders/13Z8Y9zgsh5IZaDNkkRyo1wkoMgbdUxT5?usp=sharing) **PyPSA-Earth: A Global Sector-Coupled Open-Source Multi-Energy System Model** diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 7d1f55103..2f2824a54 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -36,11 +36,15 @@ databundles: category: data destination: "data" urls: - zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_NGBJ.zip?download=1 - gdrive: https://drive.google.com/file/d/1Vb1ISjhy7iwTTZYeezGd6S4nLt-EDGme/view?usp=drive_link + zenodo: https://sandbox.zenodo.org/records/145504/files/bundle_tutorial_NGBJ_with_gadmlike.zip?download=1 + gdrive: https://drive.google.com/file/d/12K03Epx3O9o-IQLh9afzCQyT-nMKWM3P/view?usp=drive_link output: - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + - data/gadm/gadm41_NGA/gadm41_NGA.gpkg # needed in build_shapes + - data/gadm/gadm41_BEN/gadm41_BEN.gpkg # needed in build_shapes + - data/gadm/gadm36_NGA/gadm36_NGA.gpkg # needed in sector-coupled model + - data/gadm/gadm36_BEN/gadm36_BEN.gpkg # needed in sector-coupled model # tutorial bundle specific for Botswana only bundle_tutorial_BW: @@ -49,11 +53,13 @@ databundles: category: data destination: "data" urls: - zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_BW.zip?download=1 - gdrive: https://drive.google.com/file/d/19IXvTD8gVSzgTInL85ta7QjaNI8ZPCCY/view?usp=drive_link + zenodo: https://sandbox.zenodo.org/records/145504/files/bundle_tutorial_BW_with_gadmlike.zip?download=1 + gdrive: https://drive.google.com/file/d/1YbbYGs1NsSsZYqNX1g1Jo-iJzt5m-81c/view?usp=drive_link output: - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + - data/gadm/gadm41_BWA/gadm41_BWA.gpkg # needed in build_shapes + - data/gadm/gadm36_BWA/gadm36_BWA.gpkg # needed in sector-coupled model # tutorial bundle specific for Morocco only bundle_tutorial_MA: @@ -364,3 +370,32 @@ databundles: urls: protectedplanet: https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{month:s}{year:d}_Public_shp.zip output: [data/landcover/world_protected_areas/*] + + # Backup tutorial bundles with no gadm-like data; for reference: + # https://github.com/pypsa-meets-earth/pypsa-earth/issues/1258 + # + # # tutorial bundle specific for Nigeria and Benin only, without gadm-like data + # bundle_tutorial_NGBJ: + # countries: [NG, BJ] + # tutorial: true + # category: data + # destination: "data" + # urls: + # zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_NGBJ.zip?download=1 + # gdrive: https://drive.google.com/file/d/1Vb1ISjhy7iwTTZYeezGd6S4nLt-EDGme/view?usp=drive_link + # output: + # - data/gebco/GEBCO_2021_TID.nc + # - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + + # # tutorial bundle specific for Botswana only, without gadm-like data + # bundle_tutorial_BW: + # countries: [BW] + # tutorial: true + # category: data + # destination: "data" + # urls: + # zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_BW.zip?download=1 + # gdrive: https://drive.google.com/file/d/19IXvTD8gVSzgTInL85ta7QjaNI8ZPCCY/view?usp=drive_link + # output: + # - data/gebco/GEBCO_2021_TID.nc + # - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst new file mode 100644 index 000000000..68a944cb9 --- /dev/null +++ b/doc/docker_containers.rst @@ -0,0 +1,51 @@ +.. SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +.. _docker_containers: + +Alternate Installation with Docker +=============================================== + +This is an alternative way to create a development environment for PyPSA-Earth. This method is useful for users who are not familiar with programming or Python, or who do not want to install Python on their local machine. It uses Docker containers to create a development environment for PyPSA-Earth. + +This section provides a step-by-step guide on how to set up and use Docker containers to run PyPSA-Earth. + +Steps: + +1. Install Docker: Follow the instructions for your operating system: + +* `Windows `_ +* `Linux `_ +* `MacOS `_ + + Ensure Docker is installed on your system. + +2. You can use the link `here `_ to install Visual Studio Code on your operating system. Ensure to select the most compatible file for your operating system. + +3. Install GitHub Desktop for your OS `here `_. + +4. Clone the repository: + * Open GitHub Desktop. + * Click on "File" in the top left corner. + * Click on "Clone Repository". + * Paste the following URL in the URL field: + + .. code:: bash + + https://github.com/pypsa-meets-earth/pypsa-earth.git + + * Click on "Clone". + * Choose the location where you want to save the repository. + * Click on "Current Branch: main" and select `devContainers`. + * Click on "Open in Visual Studio Code". + + The repository will be cloned to your local machine. + +5. Rebuild and open in a container: + * Open the repository in VSCode. + * Click on the icon in the far bottom left corner of the VSCode window. + * Click on "Reopen in Container". + * Wait for the container to build and open the repository in the container. + + The environment will be ready for use. You can now run PyPSA-Earth in the container. diff --git a/doc/index.rst b/doc/index.rst index f9328bd09..92e17a55a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -120,6 +120,7 @@ Documentation * :doc:`introduction` * :doc:`installation` +* :doc:`docker_containers` * :doc:`short_tutorial` * :doc:`tutorial` * :doc:`data_workflow` @@ -132,6 +133,7 @@ Documentation introduction installation + docker_containers short_tutorial tutorial data_workflow diff --git a/doc/release_notes.rst b/doc/release_notes.rst index a4b36b0e3..65e933960 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -15,11 +15,16 @@ This part of documentation collects descriptive release notes to capture the mai * Include option in the config to allow for custom airport data `PR #1241 `__ +* Added Dev Containers and docker as an option to get started with pypsa-earth `PR #1228 `__ + * Add a list of PyPSA-Earth applications in academic and industrial projects `PR #1255 `__ **Minor Changes and bug-fixing** +* Added electricity bus to Fischer-Tropsch in prepare_sector_network.py `PR #1226 `__ + +* Update BW, NG and BJ tutorial databundles to include gadm-like sources from geoboundaries `PR #1257 `__ PyPSA-Earth 0.5.0 diff --git a/doc/requirements.txt b/doc/requirements.txt index d2b518fd8..1dd52eaaf 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -8,7 +8,7 @@ sphinx_book_theme sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 -pypsa +pypsa >=0.24, <0.25 vresutils>=0.3.1 powerplantmatching>=0.5.5 atlite>=0.2.9 @@ -33,7 +33,9 @@ gitpython chaospy numba ruamel.yaml<=0.17.26 -earth-osm>=0.1.0, <0.2.0 +earth-osm>=2.3.post1 reverse-geocode pyDOE2 -# graphviz +graphviz + +fake_useragent diff --git a/envs/environment.yaml b/envs/environment.yaml index b82cf3aa2..96278f6f2 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -17,9 +17,7 @@ dependencies: # currently the packages are being installed with pip # need to move back to conda once the issues will be resolved - powerplantmatching>=0.5.19 -# - earth-osm>=2.1 -# until the release will incorporate all the fixes needed -# to work with CDS beta +- earth-osm>=2.3.post1 - atlite>=0.3 # Dependencies of the workflow itself @@ -35,8 +33,8 @@ dependencies: # starting from 1.3.5 numpoly requires numpy>2.0 which leads to issues - numpoly<=1.3.4 - pandas -- geopandas>=0.11.0, <=0.14.3 -- fiona<1.10.0 +- geopandas>=0.11.0 +- fiona!=1.8.22 - xarray>=2023.11.0, <2023.12.0 - netcdf4 - networkx @@ -86,7 +84,6 @@ dependencies: - gurobi - pip: - - earth-osm==2.2 # until conda release it out for earth-osm - git+https://github.com/davide-f/google-drive-downloader@master # google drive with fix for virus scan - tsam>=1.1.0 - chaospy # lastest version only available on pip diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 3d6c73cb8..dc03380db 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -325,6 +325,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): clusters="4", ll="c1", opts="Co2L-4H", + configfile="test/config.sector.yaml", ) configure_logging(snakemake) @@ -372,10 +373,10 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): co2limit = co2limit * float(m[0]) logger.info("Setting CO2 limit according to emission base year.") elif len(m) > 0: - co2limit = float(m[0]) * snakemake.params.electricity["co2base"] + co2limit = float(m[0]) * float(snakemake.params.electricity["co2base"]) logger.info("Setting CO2 limit according to wildcard value.") else: - co2limit = snakemake.params.electricity["co2limit"] + co2limit = float(snakemake.params.electricity["co2limit"]) logger.info("Setting CO2 limit according to config value.") add_co2limit(n, co2limit, Nyears) break diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 796125b95..880dafef7 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -170,6 +170,7 @@ def H2_liquid_fossil_conversions(n, costs): bus0=spatial.nodes + " H2", bus1=spatial.oil.nodes, bus2=spatial.co2.nodes, + bus3=spatial.nodes, carrier="Fischer-Tropsch", efficiency=costs.at["Fischer-Tropsch", "efficiency"], capital_cost=costs.at["Fischer-Tropsch", "fixed"] @@ -178,6 +179,8 @@ def H2_liquid_fossil_conversions(n, costs): ], # Use efficiency to convert from EUR/MW_FT/a to EUR/MW_H2/a efficiency2=-costs.at["oil", "CO2 intensity"] * costs.at["Fischer-Tropsch", "efficiency"], + efficiency3=-costs.at["Fischer-Tropsch", "electricity-input"] + / costs.at["Fischer-Tropsch", "hydrogen-input"], p_nom_extendable=True, p_min_pu=options.get("min_part_load_fischer_tropsch", 0), lifetime=costs.at["Fischer-Tropsch", "lifetime"],