Revert "code: merge commit"

This reverts commit 1213ffb, reversing
changes made to 4363862.

.github/workflows/test.yml

@@ -70,7 +70,7 @@ jobs:
       run: micromamba list
 
     - name: Run Test
-      run: make checks
+      run: make test
 
     - name: Upload artifacts
       if: always()

Makefile

@@ -2,9 +2,9 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-.PHONY: checks tests setup clean
+.PHONY: test setup clean
 
-tests:
+test:
 	set -e
 	snakemake solve_all_networks -call --configfile config.tutorial.yaml # this runs the tutorial config
 	snakemake solve_all_networks -call --configfile config.tutorial.yaml test/config.custom.yaml # add custom config to tutorial config
@@ -14,9 +14,6 @@ tests:
 	snakemake -c4 solve_sector_networks --configfile config.tutorial.yaml test/config.test1.yaml
 	echo "All tests completed successfully."
 
-checks: tests
-	pytest test
-
 setup:
 	# Add setup commands here
 	echo "Setup complete."

REUSE.toml

@@ -19,18 +19,13 @@ precedence = "aggregate"
 SPDX-FileCopyrightText = "The PyPSA-Earth and PyPSA-Eur Authors"
 SPDX-License-Identifier = "CC-BY-4.0"
 
-[[annotations]]
-path = "test/test_data/**"
-precedence = "aggregate"
-SPDX-FileCopyrightText = "The PyPSA-Earth and PyPSA-Eur Authors"
-SPDX-License-Identifier = "CC-BY-4.0"
-
 [[annotations]]
 path = "data/**"
 precedence = "aggregate"
 SPDX-FileCopyrightText = "The PyPSA-Earth and PyPSA-Eur Authors"
 SPDX-License-Identifier = "CC-BY-4.0"
 
+
 [[annotations]]
 path = ".github/**"
 precedence = "aggregate"

Snakefile

@@ -2,14 +2,11 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-import os
-import pathlib
-import re
 import sys
-import yaml
 
 sys.path.append("./scripts")
 
+from os.path import normpath, exists, isdir
 from shutil import copyfile, move
 
 from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider
@@ -21,9 +18,9 @@ from _helpers import (
     copy_default_files,
 )
 from build_demand_profiles import get_load_paths_gegis
-from build_test_configs import create_test_config
 from retrieve_databundle_light import datafiles_retrivedatabundle
-from subprocess import run
+from pathlib import Path
+
 
 HTTP = HTTPRemoteProvider()
 
@@ -299,7 +296,7 @@ rule build_bus_regions:
         base_network="networks/" + RDIR + "base.nc",
         #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson",
         #using this line instead of the following will test updated gadm shapes for MA.
-        #To use: download file from the google drive and place it in resources/" + RDIR + "shapes/
+        #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/
         #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM
         gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson",
     output:
@@ -327,7 +324,7 @@ def terminate_if_cutout_exists(config=config):
 
     for ct in set(config_cutouts):
         cutout_fl = "cutouts/" + CDIR + ct + ".nc"
-        if pathlib.Path(cutout_fl).exists():
+        if os.path.exists(cutout_fl):
             raise Exception(
                 "An option `build_cutout` is enabled, while a cutout file '"
                 + cutout_fl
@@ -435,7 +432,7 @@ rule build_demand_profiles:
         load=load_data_paths,
         #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson",
         #using this line instead of the following will test updated gadm shapes for MA.
-        #To use: download file from the google drive and place it in resources/" + RDIR + "shapes/
+        #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/
         #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM
         gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson",
     output:
@@ -495,12 +492,15 @@ rule build_powerplants:
         gadm_layer_id=config["build_shape_options"]["gadm_layer_id"],
         alternative_clustering=config["cluster_options"]["alternative_clustering"],
         powerplants_filter=config["electricity"]["powerplants_filter"],
-        custom_powerplants=config["electricity"]["custom_powerplants"],
     input:
         base_network="networks/" + RDIR + "base.nc",
         pm_config="configs/powerplantmatching_config.yaml",
-        custom_powerplants_file="data/custom_powerplants.csv",
+        custom_powerplants="data/custom_powerplants.csv",
         osm_powerplants="resources/" + RDIR + "osm/clean/all_clean_generators.csv",
+        #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson",
+        #using this line instead of the following will test updated gadm shapes for MA.
+        #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/
+        #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM
         gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson",
     output:
         powerplants="resources/" + RDIR + "powerplants.csv",
@@ -544,7 +544,7 @@ rule add_electricity:
         powerplants="resources/" + RDIR + "powerplants.csv",
         #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson",
         #using this line instead of the following will test updated gadm shapes for MA.
-        #To use: download file from the google drive and place it in resources/" + RDIR + "shapes/
+        #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/
         #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM
         gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson",
         hydro_capacities="data/hydro_capacities.csv",
@@ -627,7 +627,7 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == True:
             + "bus_regions/regions_offshore_elec_s{simpl}.geojson",
             #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson",
             #using this line instead of the following will test updated gadm shapes for MA.
-            #To use: download file from the google drive and place it in resources/" + RDIR + "shapes/
+            #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/
             #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM
             gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson",
             # busmap=ancient('resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}.csv'),
@@ -822,7 +822,7 @@ if config["monte_carlo"]["options"].get("add_to_snakefile", False) == False:
         output:
             "results/" + RDIR + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
         log:
-            solver=os.path.normpath(
+            solver=normpath(
                 "logs/"
                 + RDIR
                 + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"
@@ -892,7 +892,7 @@ if config["monte_carlo"]["options"].get("add_to_snakefile", False) == True:
             + RDIR
             + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{unc}.nc",
         log:
-            solver=os.path.normpath(
+            solver=normpath(
                 "logs/"
                 + RDIR
                 + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{unc}_solver.log"
@@ -1046,8 +1046,6 @@ if not config["custom_data"]["gas_network"]:
             year=config["build_shape_options"]["year"],
             nprocesses=config["build_shape_options"]["nprocesses"],
             contended_flag=config["build_shape_options"]["contended_flag"],
-            gadm_file_prefix=config["build_shape_options"]["gadm_file_prefix"],
-            gadm_url_prefix=config["build_shape_options"]["gadm_url_prefix"],
             geo_crs=config["crs"]["geo_crs"],
             custom_gas_network=config["custom_data"]["gas_network"],
         input:
@@ -1069,10 +1067,6 @@ rule prepare_sector_network:
     params:
         costs=config["costs"],
         electricity=config["electricity"],
-        contended_flag=config["build_shape_options"]["contended_flag"],
-        gadm_file_prefix=config["build_shape_options"]["gadm_file_prefix"],
-        gadm_url_prefix=config["build_shape_options"]["gadm_url_prefix"],
-        geo_crs=config["crs"]["geo_crs"],
     input:
         network=RESDIR
         + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc",
@@ -1165,10 +1159,6 @@ rule add_export:
         endogenous_price=config["export"]["endogenous_price"],
         snapshots=config["snapshots"],
         costs=config["costs"],
-        contended_flag=config["build_shape_options"]["contended_flag"],
-        gadm_file_prefix=config["build_shape_options"]["gadm_file_prefix"],
-        gadm_url_prefix=config["build_shape_options"]["gadm_url_prefix"],
-        geo_crs=config["crs"]["geo_crs"],
     input:
         overrides="data/override_component_attrs",
         export_ports="resources/" + SECDIR + "export_ports.csv",
@@ -1837,10 +1827,6 @@ rule build_industrial_distribution_key:  #default data
         gadm_level=config["sector"]["gadm_level"],
         alternative_clustering=config["cluster_options"]["alternative_clustering"],
         industry_database=config["custom_data"]["industry_database"],
-        contended_flag=config["build_shape_options"]["contended_flag"],
-        gadm_file_prefix=config["build_shape_options"]["gadm_file_prefix"],
-        gadm_url_prefix=config["build_shape_options"]["gadm_url_prefix"],
-        geo_crs=config["crs"]["geo_crs"],
     input:
         regions_onshore="resources/"
         + RDIR
@@ -2145,6 +2131,10 @@ rule run_scenario:
     resources:
         mem_mb=5000,
     run:
+        from build_test_configs import create_test_config
+        import yaml
+        from subprocess import run
+
         # get base configuration file from diff config
         with open(input.diff_config) as f:
             base_config_path = (
@@ -2181,6 +2171,6 @@ rule run_all_scenarios:
             "results/{scenario_name}/scenario.done",
             scenario_name=[
                 c.stem.replace("config.", "")
-                for c in pathlib.Path("configs/scenarios").glob("config.*.yaml")
+                for c in Path("configs/scenarios").glob("config.*.yaml")
             ],
         ),

config.default.yaml

@@ -115,8 +115,6 @@ build_shape_options:
   # false (not "false") no pop addition to shape which is useful when generating only cutout
   gdp_method: "standard" # "standard" pulls from web 1x1km raster, false (not "false") no gdp addition to shape which useful when generating only cutout
   contended_flag: "set_by_country" # "set_by_country" assigns the contended areas to the countries according to the GADM database, "drop" drops these contended areas from the model
-  gadm_file_prefix: "gadm41_"
-  gadm_url_prefix: "https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/"
 
 clean_osm_data_options: # osm = OpenStreetMap
   names_by_shapes: true # Set the country name based on the extended country shapes

doc/conf.py

@@ -12,16 +12,16 @@
 # -- Path setup --------------------------------------------------------------
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
-# documentation root, use pathlib.Path.absolute to make it absolute, like shown here.
+# documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import datetime
-import pathlib
+import os
 import shutil
 import sys
 
 from git import Repo
 
-sys.path.insert(0, str(pathlib.Path("../scripts").absolute()))
+sys.path.insert(0, os.path.abspath("../scripts"))
 for p in sys.path:
     print(p)
 

scripts/add_brownfield.py

@@ -11,9 +11,11 @@
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import mock_snakemake
+import xarray as xr
 from add_existing_baseyear import add_build_year_to_new_assets
 
+# from pypsa.clustering.spatial import normed_or_uniform
+
 logger = logging.getLogger(__name__)
 idx = pd.IndexSlice
 
@@ -225,6 +227,8 @@ def disable_grid_expansion_if_limit_hit(n):
 if __name__ == "__main__":
     if "snakemake" not in globals():
 
+        from _helpers import mock_snakemake
+
         snakemake = mock_snakemake(
             "add_brownfield",
             simpl="",

scripts/add_electricity.py

@@ -84,26 +84,25 @@
 - additional open- and combined-cycle gas turbines (if ``OCGT`` and/or ``CCGT`` is listed in the config setting ``electricity: extendable_carriers``)
 """
 
+import os
+
 import numpy as np
 import pandas as pd
 import powerplantmatching as pm
 import pypsa
 import xarray as xr
-from _helpers import (
-    configure_logging,
-    create_logger,
-    mock_snakemake,
-    normed,
-    read_csv_nafix,
-    update_p_nom_max,
-)
+from _helpers import configure_logging, create_logger, read_csv_nafix, update_p_nom_max
 from powerplantmatching.export import map_country_bus
 
 idx = pd.IndexSlice
 
 logger = create_logger(__name__)
 
 
+def normed(s):
+    return s / s.sum()
+
+
 def calculate_annuity(n, r):
     """
     Calculate the annuity factor for an asset with lifetime n years and
@@ -149,6 +148,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1):
     for attr in ("investment", "lifetime", "FOM", "VOM", "efficiency", "fuel"):
         overwrites = config.get(attr)
         if overwrites is not None:
+            breakpoint()
             overwrites = pd.Series(overwrites)
             costs.loc[overwrites.index, attr] = overwrites
             logger.info(
@@ -817,6 +817,8 @@ def add_nice_carrier_names(n, config):
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
         snakemake = mock_snakemake("add_electricity")
 
     configure_logging(snakemake)

scripts/add_existing_baseyear.py

@@ -8,6 +8,7 @@
 """
 
 import logging
+import os
 from types import SimpleNamespace
 
 import country_converter as coco
@@ -16,8 +17,14 @@
 import powerplantmatching as pm
 import pypsa
 import xarray as xr
-from _helpers import mock_snakemake
-from prepare_sector_network import define_spatial, prepare_costs
+
+# from _helpers import (
+#     configure_logging,
+#     set_scenario_config,
+#     update_config_from_wildcards,
+# )
+# from add_electricity import sanitize_carriers
+from prepare_sector_network import define_spatial, prepare_costs  # , cluster_heat_buses
 
 logger = logging.getLogger(__name__)
 cc = coco.CountryConverter()
@@ -572,6 +579,8 @@ def add_heating_capacities_installed_before_baseyear(
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
         snakemake = mock_snakemake(
             "add_existing_baseyear",
             simpl="",

scripts/add_extra_components.py

@@ -52,12 +52,12 @@
 
 - ``Stores`` of carrier 'H2' and/or 'battery' in combination with ``Links``. If this option is chosen, the script adds extra buses with corresponding carrier where energy ``Stores`` are attached and which are connected to the corresponding power buses via two links, one each for charging and discharging. This leads to three investment variables for the energy capacity, charging and discharging capacity of the storage unit.
 """
-
+import os
 
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging, create_logger, mock_snakemake
+from _helpers import configure_logging, create_logger
 from add_electricity import (
     _add_missing_carriers_from_costs,
     add_nice_carrier_names,
@@ -268,6 +268,8 @@ def attach_hydrogen_pipelines(n, costs, config):
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
         snakemake = mock_snakemake("add_extra_components", simpl="", clusters=10)
 
     configure_logging(snakemake)