Merge branch 'main' into update-FT

Makefile

@@ -11,7 +11,7 @@ test:
 	snakemake solve_all_networks -call --configfile config.tutorial.yaml configs/scenarios/config.NG.yaml
 	snakemake solve_all_networks_monte -call --configfile config.tutorial.yaml test/config.monte_carlo.yaml
 	snakemake solve_all_networks -call --configfile config.tutorial.yaml test/config.landlock.yaml
-	snakemake -c4 solve_sector_networks --configfile config.tutorial.yaml test/config.test1.yaml
+	snakemake -c4 solve_sector_networks --configfile config.tutorial.yaml test/config.sector.yaml
 	echo "All tests completed successfully."
 
 setup:
@@ -24,5 +24,5 @@ clean:
 	snakemake -j1 solve_all_networks --delete-all-output --configfile config.tutorial.yaml configs/scenarios/config.NG.yaml
 	snakemake -j1 solve_all_networks_monte --delete-all-output --configfile test/config.monte_carlo.yaml
 	snakemake -j1 run_all_scenarios --delete-all-output --configfile test/config.landlock.yaml
-	snakemake -j1 solve_sector_networks --delete-all-output --configfile test/config.test1.yaml
+	snakemake -j1 solve_sector_networks --delete-all-output --configfile test/config.sector.yaml
 	echo "Clean-up complete."

config.default.yaml

@@ -107,6 +107,7 @@ cluster_options:
 
 build_shape_options:
   gadm_layer_id: 1 # GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country, 1: region/county-like, 2: municipality-like
+  simplify_gadm: true # When true, shape polygons are simplified else no
   update_file: false # When true, all the input files are downloaded again and replace the existing files
   out_logging: true # When true, logging is printed to console
   year: 2020 # reference year used to derive shapes, info on population and info on GDP

doc/configtables/build_shape_options.csv

@@ -1,9 +1,10 @@
-,Unit,Values,Description
-gadm_layer_id,, "{0, 1, 2}", "GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country, 1: region/county-like, 2: municipality-like."
-update_file, bool, "{True, False}", "True: all input files are downloaded again and replace the existing files."
-out_logging, bool, "{True, False}", "True: Logging is printed in the console."
-year,, "past year; e.g. YYYY", "Reference year used to derive shapes, info on population and info on GDP."
-nprocesses, int,, "Number of processes to be used in build_shapes."
-worldpop_method,, "{""standard"", ""api"", false}", "Specifies how population is added to every shape: ""standard"" pulls from web 1kmx1km raster; ""api"" pulls from API 100mx100m raster; false (not ""false"") no population addition to shape. This is useful when generating only cutout."
-gdp_method,, "{""standard"", false}", "Specifies how GDP is added to every shape: ""standard"" pulls from web 1x1km raster; false (not ""false"") no gdp addition to shape. This is useful when generating only cutout."
-contended_flag,, "{""set_by_country"", ""drop""}", "Specifies what to do with contended countries: ""set_by_country"" assigns the contended areas to the countries according to the GADM database; ""drop"" drops the contended areas from the model."
+,Unit,Values,Description,,,,
+gadm_layer_id,,"""{0",1,"2}""","""GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country",1: region/county-like,"2: municipality-like."""
+simplify_gadm,bool," ""{True"," False}""",True: shape polygons are simplified else no,,,
+update_file, bool," ""{True"," False}"""," ""True: all input files are downloaded again and replace the existing files.""",,,
+out_logging, bool," ""{True"," False}"""," ""True: Logging is printed in the console.""",,,
+year,," ""past year; e.g. YYYY"""," ""Reference year used to derive shapes"," info on population and info on GDP.""",,,
+nprocesses, int,," ""Number of processes to be used in build_shapes.""",,,,
+worldpop_method,," ""{""standard"""," ""api"""," false}""","""Specifies how population is added to every shape: ""standard"" pulls from web 1kmx1km raster; ""api"" pulls from API 100mx100m raster; false (not ""false"") no population addition to shape. This is useful when generating only cutout.""",,
+gdp_method,," ""{""standard"""," false}""","""Specifies how GDP is added to every shape: ""standard"" pulls from web 1x1km raster; false (not ""false"") no gdp addition to shape. This is useful when generating only cutout.""",,,
+contended_flag,," ""{""set_by_country"""," ""drop""}"""," ""Specifies what to do with contended countries: ""set_by_country"" assigns the contended areas to the countries according to the GADM database; ""drop"" drops the contended areas from the model.""",,,

doc/release_notes.rst

@@ -82,6 +82,10 @@ PyPSA-Earth 0.5.0
 
 * Drop vrestil dependency `PR #1220 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1220>`__
 
+* Include a configuration option to simplify / not simplify shapefiles based on a boolean value specified under `build_shape_options:simplify_gadm` option in the config file `PR 1138 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1138>`_
+
+* Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1134>`_
+
 * Remove duplicate entries from hydrogen export ports `PR #1233 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1233>`__
 
 * Fix the environment placing a version limit to numpoly `PR #1237 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1237>`__

scripts/add_brownfield.py

@@ -232,9 +232,9 @@ def disable_grid_expansion_if_limit_hit(n):
         snakemake = mock_snakemake(
             "add_brownfield",
             simpl="",
-            clusters="10",
-            ll="c1.0",
-            opts="Co2L",
+            clusters="4",
+            ll="c1",
+            opts="Co2L-4H",
             planning_horizons="2030",
             sopts="144H",
             discountrate=0.071,

scripts/add_existing_baseyear.py

@@ -586,11 +586,11 @@ def add_heating_capacities_installed_before_baseyear(
             simpl="",
             clusters="4",
             ll="c1",
-            opts="Co2L",
+            opts="Co2L-4H",
             planning_horizons="2030",
             sopts="144H",
             discountrate=0.071,
-            demand="DF",
+            demand="AB",
             h2export="120",
         )
 

scripts/add_export.py

@@ -211,9 +211,9 @@ def create_export_profile():
         snakemake = mock_snakemake(
             "add_export",
             simpl="",
-            clusters="10",
-            ll="c1.0",
-            opts="Co2L",
+            clusters="4",
+            ll="c1",
+            opts="Co2L-4H",
             planning_horizons="2030",
             sopts="144H",
             discountrate="0.071",

scripts/augmented_line_connections.py

@@ -55,7 +55,7 @@ def haversine(p):
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake(
-            "augmented_line_connections", network="elec", simpl="", clusters="54"
+            "augmented_line_connections", network="elec", simpl="", clusters="4"
         )
 
     configure_logging(snakemake)

scripts/build_base_energy_totals.py

@@ -357,7 +357,7 @@ def calc_sector(sector):
         snakemake = mock_snakemake(
             "build_base_energy_totals",
             simpl="",
-            clusters=19,
+            clusters=4,
             demand="AB",
             planning_horizons=2030,
         )

scripts/build_base_industry_totals.py

@@ -94,7 +94,7 @@ def create_industry_base_totals(df):
         snakemake = mock_snakemake(
             "build_base_industry_totals",
             planning_horizons=2030,
-            demand="EG",
+            demand="AB",
         )
 
     # Loading config file and wild cards

scripts/build_clustered_population_layouts.py

@@ -20,7 +20,7 @@
         snakemake = mock_snakemake(
             "build_clustered_population_layouts",
             simpl="",
-            clusters=38,
+            clusters=4,
         )
 
     cutout_path = (

scripts/build_cop_profiles.py

@@ -31,7 +31,7 @@ def coefficient_of_performance(delta_T, source="air"):
         snakemake = mock_snakemake(
             "build_cop_profiles",
             simpl="",
-            clusters=15,
+            clusters=4,
         )
 
     for area in ["total", "urban", "rural"]:

scripts/build_existing_heating_distribution.py

@@ -170,9 +170,9 @@ def build_existing_heating():
         snakemake = mock_snakemake(
             "build_existing_heating_distribution",
             simpl="",
-            clusters=4,
+            clusters="4",
             planning_horizons=2030,
-            demand="DF",
+            demand="AB",
         )
 
     build_existing_heating()

scripts/build_heat_demand.py

@@ -18,7 +18,7 @@
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
-        snakemake = mock_snakemake("build_heat_demand", simpl="", clusters="10")
+        snakemake = mock_snakemake("build_heat_demand", simpl="", clusters="4")
 
     time = pd.date_range(freq="h", **snakemake.params.snapshots)
     cutout_config = snakemake.input.cutout

scripts/build_industrial_database.py

@@ -499,12 +499,12 @@ def create_paper_df():
             "build_industrial_database",
             simpl="",
             clusters="4",
-            ll="c1.0",
-            opts="Co2L",
+            ll="c1",
+            opts="Co2L-4H",
             planning_horizons="2030",
             sopts="144H",
-            discountrate="0.071",
-            demand="DF",
+            discountrate=0.071,
+            demand="AB",
         )
 
     industrial_database_steel = create_steel_db()

scripts/build_industrial_distribution_key.py

@@ -123,9 +123,9 @@ def match_technology(df):
         snakemake = mock_snakemake(
             "build_industrial_distribution_key",
             simpl="",
-            clusters=12,
-            demand="AB",
+            clusters="4",
             planning_horizons=2050,
+            demand="AB",
         )
 
     regions = gpd.read_file(snakemake.input.regions_onshore)

scripts/build_industry_demand.py

@@ -54,7 +54,7 @@ def country_to_nodal(industrial_production, keys):
         snakemake = mock_snakemake(
             "build_industry_demand",
             simpl="",
-            clusters=10,
+            clusters="4",
             planning_horizons=2030,
             demand="AB",
         )

scripts/build_shapes.py

@@ -363,6 +363,7 @@ def eez(
     distance=0.01,
     minarea=0.01,
     tolerance=0.01,
+    simplify_gadm=True,
 ):
     """
     Creates offshore shapes by buffer smooth countryshape (=offset country
@@ -387,22 +388,26 @@ def eez(
         }
     ).set_index("name")
 
-    ret_df = ret_df.geometry.map(
-        lambda x: _simplify_polys(x, minarea=minarea, tolerance=tolerance)
-    )
+    if simplify_gadm:
+        ret_df = ret_df.geometry.map(
+            lambda x: _simplify_polys(x, minarea=minarea, tolerance=tolerance)
+        )
 
-    ret_df = ret_df.apply(lambda x: make_valid(x))
+        ret_df = ret_df.apply(lambda x: make_valid(x))
 
     country_shapes_with_buffer = country_shapes.buffer(distance)
     ret_df_new = ret_df.difference(country_shapes_with_buffer)
 
-    # repeat to simplify after the buffer correction
-    ret_df_new = ret_df_new.map(
-        lambda x: (
-            x if x is None else _simplify_polys(x, minarea=minarea, tolerance=tolerance)
+    if simplify_gadm:
+        # repeat to simplify after the buffer correction
+        ret_df_new = ret_df_new.map(
+            lambda x: (
+                x
+                if x is None
+                else _simplify_polys(x, minarea=minarea, tolerance=tolerance)
+            )
         )
-    )
-    ret_df_new = ret_df_new.apply(lambda x: x if x is None else make_valid(x))
+        ret_df_new = ret_df_new.apply(lambda x: x if x is None else make_valid(x))
 
     # Drops empty geometry
     ret_df = ret_df_new.dropna()
@@ -1253,6 +1258,7 @@ def gadm(
     out_logging=False,
     year=2020,
     nprocesses=None,
+    simplify_gadm=True,
 ):
     if out_logging:
         logger.info("Stage 3 of 5: Creation GADM GeoDataFrame")
@@ -1302,7 +1308,9 @@ def gadm(
         lambda x: x if x.find(".") == 0 else "." + x
     )
     df_gadm.set_index("GADM_ID", inplace=True)
-    df_gadm["geometry"] = df_gadm["geometry"].map(_simplify_polys)
+
+    if simplify_gadm:
+        df_gadm["geometry"] = df_gadm["geometry"].map(_simplify_polys)
     df_gadm.geometry = df_gadm.geometry.apply(
         lambda r: make_valid(r) if not r.is_valid else r
     )
@@ -1335,6 +1343,7 @@ def gadm(
     contended_flag = snakemake.params.build_shape_options["contended_flag"]
     worldpop_method = snakemake.params.build_shape_options["worldpop_method"]
     gdp_method = snakemake.params.build_shape_options["gdp_method"]
+    simplify_gadm = snakemake.params.build_shape_options["simplify_gadm"]
 
     country_shapes = countries(
         countries_list,
@@ -1346,7 +1355,7 @@ def gadm(
     country_shapes.to_file(snakemake.output.country_shapes)
 
     offshore_shapes = eez(
-        countries_list, geo_crs, country_shapes, EEZ_gpkg, out_logging
+        countries_list, geo_crs, country_shapes, EEZ_gpkg, out_logging, simplify_gadm
     )
 
     offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes)
@@ -1368,5 +1377,6 @@ def gadm(
         out_logging,
         year,
         nprocesses=nprocesses,
+        simplify_gadm=simplify_gadm,
     )
     save_to_geojson(gadm_shapes, out.gadm_shapes)

scripts/build_solar_thermal_profiles.py

@@ -21,7 +21,7 @@
         snakemake = mock_snakemake(
             "build_solar_thermal_profiles",
             simpl="",
-            clusters=15,
+            clusters="4",
         )
 
     config = snakemake.params.solar_thermal_config

scripts/build_temperature_profiles.py

@@ -20,7 +20,7 @@
         snakemake = mock_snakemake(
             "build_temperature_profiles",
             simpl="",
-            clusters=900,
+            clusters="4",
         )
 
     time = pd.date_range(freq="h", **snakemake.params.snapshots)

scripts/cluster_network.py

@@ -657,7 +657,7 @@ def cluster_regions(busmaps, inputs, output):
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake(
-            "cluster_network", network="elec", simpl="", clusters="min"
+            "cluster_network", network="elec", simpl="", clusters="4"
         )
     configure_logging(snakemake)
 

scripts/make_summary.py

@@ -539,9 +539,9 @@ def to_csv(dfs, dir):
         snakemake = mock_snakemake(
             "make_summary",
             simpl="",
-            clusters="5",
+            clusters="4",
             ll="copt",
-            opts="Co2L-3H",
+            opts="Co2L-4H",
             country="all",
         )
         network_dir = ".."

scripts/monte_carlo.py

@@ -353,9 +353,9 @@ def validate_parameters(
         snakemake = mock_snakemake(
             "monte_carlo",
             simpl="",
-            clusters="10",
+            clusters="4",
             ll="copt",
-            opts="Co2L-24H",
+            opts="Co2L-4H",
             unc="m0",
         )
     configure_logging(snakemake)

scripts/override_respot.py

@@ -73,13 +73,13 @@ def override_values(tech, year, dr):
         snakemake = mock_snakemake(
             "override_respot",
             simpl="",
-            clusters="16",
-            ll="c1.0",
-            opts="Co2L",
+            clusters="4",
+            ll="c1",
+            opts="Co2L-4H",
             planning_horizons="2030",
-            sopts="3H",
-            demand="AP",
+            sopts="144H",
             discountrate=0.071,
+            demand="AB",
         )
 
     overrides = override_component_attrs(snakemake.input.overrides)

scripts/plot_network.py

@@ -1077,9 +1077,9 @@ def plot_sector_map(
             "plot_network",
             network="elec",
             simpl="",
-            clusters="100",
-            ll="copt",
-            opts="Co2L-3H",
+            clusters="4",
+            ll="c1",
+            opts="Co2L-4H",
             attr="p_nom",
             ext="pdf",
         )

scripts/plot_summary.py

@@ -224,9 +224,9 @@ def plot_energy(infn, snmk, fn=None):
             summary="energy",
             network="elec",
             simpl="",
-            clusters=10,
-            ll="copt",
-            opts="Co2L-24H",
+            clusters="4",
+            ll="c1",
+            opts="Co2L-4H",
             attr="",
             ext="pdf",
             country="all",