From a549e0a8a0e18d74772679518e9d064bc2099d38 Mon Sep 17 00:00:00 2001 From: Fabrizio Finozzi Date: Fri, 13 Dec 2024 15:04:38 +0100 Subject: [PATCH] Revert "code: merge commit" This reverts commit 1213ffb24df994162f0dad012529966f267c8b91, reversing changes made to 4363862f7d6d21bf605d2627788058330f0d5761. --- .github/workflows/test.yml | 2 +- Makefile | 7 +- REUSE.toml | 7 +- Snakefile | 50 +- config.default.yaml | 2 - doc/conf.py | 6 +- scripts/add_brownfield.py | 6 +- scripts/add_electricity.py | 18 +- scripts/add_existing_baseyear.py | 13 +- scripts/add_extra_components.py | 6 +- scripts/augmented_line_connections.py | 5 +- scripts/base_network.py | 105 ++- scripts/build_base_energy_totals.py | 32 +- scripts/build_base_industry_totals.py | 28 +- scripts/build_bus_regions.py | 14 +- scripts/build_clustered_population_layouts.py | 10 +- scripts/build_cop_profiles.py | 3 +- scripts/build_cutout.py | 5 +- scripts/build_demand_profiles.py | 24 +- .../build_existing_heating_distribution.py | 4 +- scripts/build_heat_demand.py | 5 +- scripts/build_industrial_database.py | 4 +- scripts/build_industrial_distribution_key.py | 48 +- scripts/build_industry_demand.py | 7 +- scripts/build_natura_raster.py | 13 +- scripts/build_osm_network.py | 71 +- scripts/build_powerplants.py | 231 ++++--- scripts/build_shapes.py | 343 +++++++--- scripts/build_ship_profile.py | 6 +- scripts/build_solar_thermal_profiles.py | 4 +- scripts/build_temperature_profiles.py | 4 +- scripts/build_test_configs.py | 9 +- scripts/clean_osm_data.py | 15 +- scripts/cluster_network.py | 24 +- scripts/copy_config.py | 16 +- scripts/download_osm_data.py | 28 +- scripts/make_statistics.py | 33 +- scripts/make_summary.py | 25 +- scripts/monte_carlo.py | 70 +- scripts/non_workflow/zenodo_handler.py | 6 +- scripts/non_workflow/zip_folder.py | 5 +- scripts/override_respot.py | 7 + scripts/plot_network.py | 13 +- scripts/plot_summary.py | 6 +- scripts/prepare_airports.py | 8 +- scripts/prepare_db.py | 5 +- scripts/prepare_energy_totals.py | 22 +- scripts/prepare_heat_data.py | 4 + scripts/prepare_network.py | 53 +- scripts/prepare_transport_data.py | 5 +- scripts/prepare_transport_data_input.py | 19 +- scripts/prepare_urban_percent.py | 17 +- scripts/retrieve_databundle_light.py | 130 ++-- scripts/simplify_network.py | 4 +- scripts/solve_network.py | 14 +- test/__init__.py | 4 - test/conftest.py | 52 -- test/test_add_extra_components.py | 137 ---- test/test_base_network.py | 491 -------------- test/test_build_demand_profiles.py | 39 -- test/test_build_powerplants.py | 135 ---- test/test_build_shapes.py | 312 --------- test/test_data/custom_NG_powerplants.csv | 5 - test/test_helpers.py | 616 ------------------ test/test_prepare_network.py | 233 ------- 65 files changed, 1041 insertions(+), 2604 deletions(-) delete mode 100644 test/__init__.py delete mode 100644 test/conftest.py delete mode 100644 test/test_add_extra_components.py delete mode 100644 test/test_base_network.py delete mode 100644 test/test_build_demand_profiles.py delete mode 100644 test/test_build_powerplants.py delete mode 100644 test/test_build_shapes.py delete mode 100644 test/test_data/custom_NG_powerplants.csv delete mode 100644 test/test_helpers.py delete mode 100644 test/test_prepare_network.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 427874ff3..a243a304b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,7 @@ jobs: run: micromamba list - name: Run Test - run: make checks + run: make test - name: Upload artifacts if: always() diff --git a/Makefile b/Makefile index 07db5cb3a..b01a61800 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later -.PHONY: checks tests setup clean +.PHONY: test setup clean -tests: +test: set -e snakemake solve_all_networks -call --configfile config.tutorial.yaml # this runs the tutorial config snakemake solve_all_networks -call --configfile config.tutorial.yaml test/config.custom.yaml # add custom config to tutorial config @@ -14,9 +14,6 @@ tests: snakemake -c4 solve_sector_networks --configfile config.tutorial.yaml test/config.test1.yaml echo "All tests completed successfully." -checks: tests - pytest test - setup: # Add setup commands here echo "Setup complete." diff --git a/REUSE.toml b/REUSE.toml index 06bce5d3b..79c9a3fb9 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -19,18 +19,13 @@ precedence = "aggregate" SPDX-FileCopyrightText = "The PyPSA-Earth and PyPSA-Eur Authors" SPDX-License-Identifier = "CC-BY-4.0" -[[annotations]] -path = "test/test_data/**" -precedence = "aggregate" -SPDX-FileCopyrightText = "The PyPSA-Earth and PyPSA-Eur Authors" -SPDX-License-Identifier = "CC-BY-4.0" - [[annotations]] path = "data/**" precedence = "aggregate" SPDX-FileCopyrightText = "The PyPSA-Earth and PyPSA-Eur Authors" SPDX-License-Identifier = "CC-BY-4.0" + [[annotations]] path = ".github/**" precedence = "aggregate" diff --git a/Snakefile b/Snakefile index 2f9bc08ce..767833df7 100644 --- a/Snakefile +++ b/Snakefile @@ -2,14 +2,11 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later -import os -import pathlib -import re import sys -import yaml sys.path.append("./scripts") +from os.path import normpath, exists, isdir from shutil import copyfile, move from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider @@ -21,9 +18,9 @@ from _helpers import ( copy_default_files, ) from build_demand_profiles import get_load_paths_gegis -from build_test_configs import create_test_config from retrieve_databundle_light import datafiles_retrivedatabundle -from subprocess import run +from pathlib import Path + HTTP = HTTPRemoteProvider() @@ -299,7 +296,7 @@ rule build_bus_regions: base_network="networks/" + RDIR + "base.nc", #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", #using this line instead of the following will test updated gadm shapes for MA. - #To use: download file from the google drive and place it in resources/" + RDIR + "shapes/ + #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson", output: @@ -327,7 +324,7 @@ def terminate_if_cutout_exists(config=config): for ct in set(config_cutouts): cutout_fl = "cutouts/" + CDIR + ct + ".nc" - if pathlib.Path(cutout_fl).exists(): + if os.path.exists(cutout_fl): raise Exception( "An option `build_cutout` is enabled, while a cutout file '" + cutout_fl @@ -435,7 +432,7 @@ rule build_demand_profiles: load=load_data_paths, #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", #using this line instead of the following will test updated gadm shapes for MA. - #To use: download file from the google drive and place it in resources/" + RDIR + "shapes/ + #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson", output: @@ -495,12 +492,15 @@ rule build_powerplants: gadm_layer_id=config["build_shape_options"]["gadm_layer_id"], alternative_clustering=config["cluster_options"]["alternative_clustering"], powerplants_filter=config["electricity"]["powerplants_filter"], - custom_powerplants=config["electricity"]["custom_powerplants"], input: base_network="networks/" + RDIR + "base.nc", pm_config="configs/powerplantmatching_config.yaml", - custom_powerplants_file="data/custom_powerplants.csv", + custom_powerplants="data/custom_powerplants.csv", osm_powerplants="resources/" + RDIR + "osm/clean/all_clean_generators.csv", + #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", + #using this line instead of the following will test updated gadm shapes for MA. + #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ + #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson", output: powerplants="resources/" + RDIR + "powerplants.csv", @@ -544,7 +544,7 @@ rule add_electricity: powerplants="resources/" + RDIR + "powerplants.csv", #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", #using this line instead of the following will test updated gadm shapes for MA. - #To use: download file from the google drive and place it in resources/" + RDIR + "shapes/ + #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson", hydro_capacities="data/hydro_capacities.csv", @@ -627,7 +627,7 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == True: + "bus_regions/regions_offshore_elec_s{simpl}.geojson", #gadm_shapes="resources/" + RDIR + "shapes/MAR2.geojson", #using this line instead of the following will test updated gadm shapes for MA. - #To use: download file from the google drive and place it in resources/" + RDIR + "shapes/ + #To use: downlaod file from the google drive and place it in resources/" + RDIR + "shapes/ #Link: https://drive.google.com/drive/u/1/folders/1dkW1wKBWvSY4i-XEuQFFBj242p0VdUlM gadm_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson", # busmap=ancient('resources/" + RDIR + "bus_regions/busmap_elec_s{simpl}.csv'), @@ -822,7 +822,7 @@ if config["monte_carlo"]["options"].get("add_to_snakefile", False) == False: output: "results/" + RDIR + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: - solver=os.path.normpath( + solver=normpath( "logs/" + RDIR + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log" @@ -892,7 +892,7 @@ if config["monte_carlo"]["options"].get("add_to_snakefile", False) == True: + RDIR + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{unc}.nc", log: - solver=os.path.normpath( + solver=normpath( "logs/" + RDIR + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{unc}_solver.log" @@ -1046,8 +1046,6 @@ if not config["custom_data"]["gas_network"]: year=config["build_shape_options"]["year"], nprocesses=config["build_shape_options"]["nprocesses"], contended_flag=config["build_shape_options"]["contended_flag"], - gadm_file_prefix=config["build_shape_options"]["gadm_file_prefix"], - gadm_url_prefix=config["build_shape_options"]["gadm_url_prefix"], geo_crs=config["crs"]["geo_crs"], custom_gas_network=config["custom_data"]["gas_network"], input: @@ -1069,10 +1067,6 @@ rule prepare_sector_network: params: costs=config["costs"], electricity=config["electricity"], - contended_flag=config["build_shape_options"]["contended_flag"], - gadm_file_prefix=config["build_shape_options"]["gadm_file_prefix"], - gadm_url_prefix=config["build_shape_options"]["gadm_url_prefix"], - geo_crs=config["crs"]["geo_crs"], input: network=RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", @@ -1165,10 +1159,6 @@ rule add_export: endogenous_price=config["export"]["endogenous_price"], snapshots=config["snapshots"], costs=config["costs"], - contended_flag=config["build_shape_options"]["contended_flag"], - gadm_file_prefix=config["build_shape_options"]["gadm_file_prefix"], - gadm_url_prefix=config["build_shape_options"]["gadm_url_prefix"], - geo_crs=config["crs"]["geo_crs"], input: overrides="data/override_component_attrs", export_ports="resources/" + SECDIR + "export_ports.csv", @@ -1837,10 +1827,6 @@ rule build_industrial_distribution_key: #default data gadm_level=config["sector"]["gadm_level"], alternative_clustering=config["cluster_options"]["alternative_clustering"], industry_database=config["custom_data"]["industry_database"], - contended_flag=config["build_shape_options"]["contended_flag"], - gadm_file_prefix=config["build_shape_options"]["gadm_file_prefix"], - gadm_url_prefix=config["build_shape_options"]["gadm_url_prefix"], - geo_crs=config["crs"]["geo_crs"], input: regions_onshore="resources/" + RDIR @@ -2145,6 +2131,10 @@ rule run_scenario: resources: mem_mb=5000, run: + from build_test_configs import create_test_config + import yaml + from subprocess import run + # get base configuration file from diff config with open(input.diff_config) as f: base_config_path = ( @@ -2181,6 +2171,6 @@ rule run_all_scenarios: "results/{scenario_name}/scenario.done", scenario_name=[ c.stem.replace("config.", "") - for c in pathlib.Path("configs/scenarios").glob("config.*.yaml") + for c in Path("configs/scenarios").glob("config.*.yaml") ], ), diff --git a/config.default.yaml b/config.default.yaml index 8c2d591cc..cd8850723 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -115,8 +115,6 @@ build_shape_options: # false (not "false") no pop addition to shape which is useful when generating only cutout gdp_method: "standard" # "standard" pulls from web 1x1km raster, false (not "false") no gdp addition to shape which useful when generating only cutout contended_flag: "set_by_country" # "set_by_country" assigns the contended areas to the countries according to the GADM database, "drop" drops these contended areas from the model - gadm_file_prefix: "gadm41_" - gadm_url_prefix: "https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/" clean_osm_data_options: # osm = OpenStreetMap names_by_shapes: true # Set the country name based on the extended country shapes diff --git a/doc/conf.py b/doc/conf.py index a56f1958a..6f1edc018 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,16 +12,16 @@ # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the -# documentation root, use pathlib.Path.absolute to make it absolute, like shown here. +# documentation root, use os.path.abspath to make it absolute, like shown here. # import datetime -import pathlib +import os import shutil import sys from git import Repo -sys.path.insert(0, str(pathlib.Path("../scripts").absolute())) +sys.path.insert(0, os.path.abspath("../scripts")) for p in sys.path: print(p) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index ce5e243f8..fba41d327 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -11,9 +11,11 @@ import numpy as np import pandas as pd import pypsa -from _helpers import mock_snakemake +import xarray as xr from add_existing_baseyear import add_build_year_to_new_assets +# from pypsa.clustering.spatial import normed_or_uniform + logger = logging.getLogger(__name__) idx = pd.IndexSlice @@ -225,6 +227,8 @@ def disable_grid_expansion_if_limit_hit(n): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "add_brownfield", simpl="", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 32b478019..75ae9ce42 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -84,19 +84,14 @@ - additional open- and combined-cycle gas turbines (if ``OCGT`` and/or ``CCGT`` is listed in the config setting ``electricity: extendable_carriers``) """ +import os + import numpy as np import pandas as pd import powerplantmatching as pm import pypsa import xarray as xr -from _helpers import ( - configure_logging, - create_logger, - mock_snakemake, - normed, - read_csv_nafix, - update_p_nom_max, -) +from _helpers import configure_logging, create_logger, read_csv_nafix, update_p_nom_max from powerplantmatching.export import map_country_bus idx = pd.IndexSlice @@ -104,6 +99,10 @@ logger = create_logger(__name__) +def normed(s): + return s / s.sum() + + def calculate_annuity(n, r): """ Calculate the annuity factor for an asset with lifetime n years and @@ -149,6 +148,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1): for attr in ("investment", "lifetime", "FOM", "VOM", "efficiency", "fuel"): overwrites = config.get(attr) if overwrites is not None: + breakpoint() overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites logger.info( @@ -817,6 +817,8 @@ def add_nice_carrier_names(n, config): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("add_electricity") configure_logging(snakemake) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 61e0d560c..f2529587d 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -8,6 +8,7 @@ """ import logging +import os from types import SimpleNamespace import country_converter as coco @@ -16,8 +17,14 @@ import powerplantmatching as pm import pypsa import xarray as xr -from _helpers import mock_snakemake -from prepare_sector_network import define_spatial, prepare_costs + +# from _helpers import ( +# configure_logging, +# set_scenario_config, +# update_config_from_wildcards, +# ) +# from add_electricity import sanitize_carriers +from prepare_sector_network import define_spatial, prepare_costs # , cluster_heat_buses logger = logging.getLogger(__name__) cc = coco.CountryConverter() @@ -572,6 +579,8 @@ def add_heating_capacities_installed_before_baseyear( if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "add_existing_baseyear", simpl="", diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 3d29976f5..2c317c305 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -52,12 +52,12 @@ - ``Stores`` of carrier 'H2' and/or 'battery' in combination with ``Links``. If this option is chosen, the script adds extra buses with corresponding carrier where energy ``Stores`` are attached and which are connected to the corresponding power buses via two links, one each for charging and discharging. This leads to three investment variables for the energy capacity, charging and discharging capacity of the storage unit. """ - +import os import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, create_logger, mock_snakemake +from _helpers import configure_logging, create_logger from add_electricity import ( _add_missing_carriers_from_costs, add_nice_carrier_names, @@ -268,6 +268,8 @@ def attach_hydrogen_pipelines(n, costs, config): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("add_extra_components", simpl="", clusters=10) configure_logging(snakemake) diff --git a/scripts/augmented_line_connections.py b/scripts/augmented_line_connections.py index 433d2f3fc..634d10eea 100644 --- a/scripts/augmented_line_connections.py +++ b/scripts/augmented_line_connections.py @@ -28,12 +28,13 @@ Description ----------- """ +import os import networkx as nx import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, create_logger, mock_snakemake +from _helpers import configure_logging, create_logger from add_electricity import load_costs from networkx.algorithms import complement from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation @@ -51,6 +52,8 @@ def haversine(p): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "augmented_line_connections", network="elec", simpl="", clusters="54" ) diff --git a/scripts/base_network.py b/scripts/base_network.py index 83c5a4830..65d640d44 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -55,26 +55,29 @@ Description ----------- """ +import os import geopandas as gpd import networkx as nx import numpy as np import pandas as pd import pypsa +import scipy as sp import shapely.prepared import shapely.wkt -from _helpers import ( - configure_logging, - create_logger, - get_path_size, - mock_snakemake, - read_csv_nafix, -) +from _helpers import configure_logging, create_logger, read_csv_nafix from shapely.ops import unary_union logger = create_logger(__name__) +def _get_oid(df): + if "tags" in df.columns: + return df.tags.str.extract('"oid"=>"(\\d+)"', expand=False) + else: + return pd.Series(np.nan, df.index) + + def get_country(df): if "tags" in df.columns: return df.tags.str.extract('"country"=>"([A-Z]{2})"', expand=False) @@ -82,6 +85,28 @@ def get_country(df): return pd.Series(np.nan, df.index) +def _find_closest_links(links, new_links, distance_upper_bound=1.5): + treecoords = np.asarray( + [np.asarray(shapely.wkt.loads(s))[[0, -1]].flatten() for s in links.geometry] + ) + querycoords = np.vstack( + [new_links[["x1", "y1", "x2", "y2"]], new_links[["x2", "y2", "x1", "y1"]]] + ) + tree = sp.spatial.KDTree(treecoords) + dist, ind = tree.query(querycoords, distance_upper_bound=distance_upper_bound) + found_b = ind < len(links) + found_i = np.arange(len(new_links) * 2)[found_b] % len(new_links) + + return ( + pd.DataFrame( + dict(D=dist[found_b], i=links.index[ind[found_b] % len(links)]), + index=new_links.index[found_i], + ) + .sort_values(by="D")[lambda ds: ~ds.index.duplicated(keep="first")] + .sort_index()["i"] + ) + + def _load_buses_from_osm(fp_buses): buses = ( read_csv_nafix(fp_buses, dtype=dict(bus_id="str", voltage="float")) @@ -102,6 +127,20 @@ def _load_buses_from_osm(fp_buses): return buses +def add_underwater_links(n, fp_offshore_shapes): + if not hasattr(n.links, "geometry"): + n.links["underwater_fraction"] = 0.0 + else: + offshore_shape = gpd.read_file(fp_offshore_shapes).unary_union + if offshore_shape is None or offshore_shape.is_empty: + n.links["underwater_fraction"] = 0.0 + else: + links = gpd.GeoSeries(n.links.geometry.dropna().map(shapely.wkt.loads)) + n.links["underwater_fraction"] = ( + links.intersection(offshore_shape).length / links.length + ) + + def _set_dc_underwater_fraction(lines_or_links, fp_offshore_shapes): # HVDC part always has some links as converters # excluding probably purely DC networks which are currently somewhat exotic @@ -155,13 +194,44 @@ def _load_lines_from_osm(fp_osm_lines): lines["length"] /= 1e3 # m to km conversion lines["v_nom"] /= 1e3 # V to kV conversion lines = lines.loc[:, ~lines.columns.str.contains("^Unnamed")] # remove unnamed col + # lines = _remove_dangling_branches(lines, buses) # TODO: add dangling branch removal? return lines -def _load_converters_from_osm(fp_osm_converters): +# TODO Seems to be not needed anymore +def _load_links_from_osm(fp_osm_converters, base_network_config, voltages_config): # the links file can be empty - if get_path_size(fp_osm_converters) == 0: + if os.path.getsize(fp_osm_converters) == 0: + links = pd.DataFrame() + return links + + links = ( + read_csv_nafix( + fp_osm_converters, + dtype=dict( + line_id="str", + bus0="str", + bus1="str", + underground="bool", + under_construction="bool", + ), + ) + .set_index("line_id") + .rename(columns=dict(voltage="v_nom", circuits="num_parallel")) + ) + + links["length"] /= 1e3 # m to km conversion + links["v_nom"] /= 1e3 # V to kV conversion + links = links.loc[:, ~links.columns.str.contains("^Unnamed")] # remove unnamed col + # links = _remove_dangling_branches(links, buses) # TODO: add dangling branch removal? + + return links + + +def _load_converters_from_osm(fp_osm_converters, buses): + # the links file can be empty + if os.path.getsize(fp_osm_converters) == 0: converters = pd.DataFrame() return converters @@ -170,13 +240,15 @@ def _load_converters_from_osm(fp_osm_converters): dtype=dict(converter_id="str", bus0="str", bus1="str"), ).set_index("converter_id") + # converters = _remove_dangling_branches(converters, buses) + converters["carrier"] = "B2B" converters["dc"] = True return converters -def _load_transformers_from_osm(fp_osm_transformers): +def _load_transformers_from_osm(fp_osm_transformers, buses): transformers = ( read_csv_nafix( fp_osm_transformers, @@ -185,6 +257,7 @@ def _load_transformers_from_osm(fp_osm_transformers): .rename(columns=dict(line_id="transformer_id")) .set_index("transformer_id") ) + # transformers = _remove_dangling_branches(transformers, buses) # TODO: add dangling branch removal? return transformers @@ -326,6 +399,12 @@ def _set_lines_s_nom_from_linetypes(n): ) * n.lines.eval("v_nom * num_parallel") +def _remove_dangling_branches(branches, buses): + return pd.DataFrame( + branches.loc[branches.bus0.isin(buses.index) & branches.bus1.isin(buses.index)] + ) + + def _set_countries_and_substations(inputs, base_network_config, countries_config, n): countries = countries_config country_shapes = gpd.read_file(inputs.country_shapes).set_index("name")["geometry"] @@ -407,8 +486,8 @@ def base_network( ): buses = _load_buses_from_osm(inputs.osm_buses).reset_index(drop=True) lines = _load_lines_from_osm(inputs.osm_lines).reset_index(drop=True) - transformers = _load_transformers_from_osm(inputs.osm_transformers) - converters = _load_converters_from_osm(inputs.osm_converters) + transformers = _load_transformers_from_osm(inputs.osm_transformers, buses) + converters = _load_converters_from_osm(inputs.osm_converters, buses) lines_ac = lines[lines.tag_frequency.astype(float) != 0].copy() lines_dc = lines[lines.tag_frequency.astype(float) == 0].copy() @@ -475,6 +554,8 @@ def base_network( if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("base_network") configure_logging(snakemake) diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py index e4f6998ad..4ac1f7889 100644 --- a/scripts/build_base_energy_totals.py +++ b/scripts/build_base_energy_totals.py @@ -6,14 +6,20 @@ # -*- coding: utf-8 -*- import glob import logging +import os +import sys from io import BytesIO +from pathlib import Path from urllib.request import urlopen from zipfile import ZipFile import country_converter as coco +import matplotlib.pyplot as plt import numpy as np import pandas as pd -from _helpers import BASE_DIR, aggregate_fuels, get_conv_factors, get_path, mock_snakemake, modify_commodity +import py7zr +import requests +from _helpers import BASE_DIR, aggregate_fuels, get_conv_factors _logger = logging.getLogger(__name__) @@ -346,6 +352,8 @@ def calc_sector(sector): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_base_energy_totals", simpl="", @@ -364,15 +372,12 @@ def calc_sector(sector): df = df.to_dict("dict") d = df["Link"] - demand_base_path = get_path(BASE_DIR, "data", "demand", "unsd", "data") - demand_path = get_path(demand_base_path, "*.txt") - if snakemake.params.update_data: # Delete and existing files to avoid duplication and double counting - files = glob.glob(str(demand_path)) + files = glob.glob(os.path.join(BASE_DIR, "data/demand/unsd/data/*.txt")) for f in files: - get_path(f).unlink(missing_ok=True) + os.remove(f) # Feed the dictionary of links to the for loop, download and unzip all files for key, value in d.items(): @@ -380,10 +385,14 @@ def calc_sector(sector): with urlopen(zipurl) as zipresp: with ZipFile(BytesIO(zipresp.read())) as zfile: - zfile.extractall(str(demand_base_path)) + zfile.extractall(os.path.join(BASE_DIR, "data/demand/unsd/data")) + + path = os.path.join(BASE_DIR, "data/demand/unsd/data") # Get the files from the path provided in the OP - all_files = list(demand_base_path.glob("*.txt")) + all_files = list( + Path(os.path.join(BASE_DIR, "data/demand/unsd/data")).glob("*.txt") + ) # Create a dataframe from all downloaded files df = pd.concat( @@ -395,9 +404,6 @@ def calc_sector(sector): " - ", expand=True ) - # Modify the commodity column, replacing typos and case-folding the strings - df["Commodity"] = df["Commodity"].map(modify_commodity) - # Remove Foootnote and Estimate from 'Commodity - Transaction' column df = df.loc[df["Commodity - Transaction"] != "Footnote"] df = df.loc[df["Commodity - Transaction"] != "Estimate"] @@ -429,7 +435,9 @@ def calc_sector(sector): df_yr = df_yr[df_yr.country.isin(countries)] # Create an empty dataframe for energy_totals_base - energy_totals_cols = pd.read_csv(get_path(BASE_DIR, "data", "energy_totals_DF_2030.csv")).columns + energy_totals_cols = pd.read_csv( + os.path.join(BASE_DIR, "data/energy_totals_DF_2030.csv") + ).columns energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries) # Lists that combine the different fuels in the dataset to the model's carriers diff --git a/scripts/build_base_industry_totals.py b/scripts/build_base_industry_totals.py index 8320f99cc..eef58f10a 100644 --- a/scripts/build_base_industry_totals.py +++ b/scripts/build_base_industry_totals.py @@ -8,16 +8,17 @@ @author: user """ + +import os +import re +from pathlib import Path + import country_converter as coco import pandas as pd -from _helpers import ( - aggregate_fuels, - get_conv_factors, - get_path, - mock_snakemake, - modify_commodity, - read_csv_nafix, -) +from _helpers import aggregate_fuels, get_conv_factors, read_csv_nafix +from prepare_sector_network import get + +# def calc_industry_base(df): def calculate_end_values(df): @@ -88,6 +89,8 @@ def create_industry_base_totals(df): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_base_industry_totals", planning_horizons=2030, @@ -114,12 +117,12 @@ def create_industry_base_totals(df): renaming_dit = transaction.set_index("Transaction")["clean_name"].to_dict() clean_industry_list = list(transaction.clean_name.unique()) - unsd_path = get_path( - get_path(snakemake.input["energy_totals_base"]).parent, "demand/unsd/data/" + unsd_path = ( + os.path.dirname(snakemake.input["energy_totals_base"]) + "/demand/unsd/data/" ) # Get the files from the path provided in the OP - all_files = list(get_path(unsd_path).glob("*.txt")) + all_files = list(Path(unsd_path).glob("*.txt")) # Create a dataframe from all downloaded files df = pd.concat( @@ -131,9 +134,6 @@ def create_industry_base_totals(df): " - ", expand=True ) - # Modify the commodity column, replacing typos and case-folding the strings - df["Commodity"] = df["Commodity"].map(modify_commodity) - df = df[ df.Commodity != "Other bituminous coal" ] # dropping problematic column leading to double counting diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 0af9bb4e7..1a0dc2338 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -42,14 +42,12 @@ Description ----------- """ +import os import geopandas as gpd -import numpy as np import pandas as pd import pypsa -from _helpers import REGION_COLS, configure_logging, create_logger, mock_snakemake -from scipy.spatial import Voronoi -from shapely.geometry import Polygon +from _helpers import REGION_COLS, configure_logging, create_logger logger = create_logger(__name__) @@ -69,8 +67,14 @@ def custom_voronoi_partition_pts(points, outline, add_bounds_shape=True, multipl polygons : N - ndarray[dtype=Polygon|MultiPolygon] """ + import numpy as np + from scipy.spatial import Voronoi + from shapely.geometry import Polygon + points = np.asarray(points) + polygons_arr = [] + if len(points) == 1: polygons_arr = [outline] else: @@ -144,6 +148,8 @@ def get_gadm_shape( if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("build_bus_regions") configure_logging(snakemake) diff --git a/scripts/build_clustered_population_layouts.py b/scripts/build_clustered_population_layouts.py index c88c50b49..374edd448 100644 --- a/scripts/build_clustered_population_layouts.py +++ b/scripts/build_clustered_population_layouts.py @@ -5,23 +5,29 @@ """ Build clustered population layouts. """ +import os import atlite import geopandas as gpd import pandas as pd import xarray as xr -from _helpers import mock_snakemake, to_csv_nafix +from _helpers import read_csv_nafix, to_csv_nafix if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_clustered_population_layouts", simpl="", clusters=38, ) - cutout_path = snakemake.input.cutout + cutout_path = ( + snakemake.input.cutout + ) # os.path.abspath(snakemake.config["atlite"]["cutout"]) cutout = atlite.Cutout(cutout_path) + # cutout = atlite.Cutout(snakemake.config['atlite']['cutout']) clustered_regions = ( gpd.read_file(snakemake.input.regions_onshore) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index afff7957c..d785b3ee6 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -7,7 +7,6 @@ """ import xarray as xr -from _helpers import mock_snakemake def coefficient_of_performance(delta_T, source="air"): @@ -27,6 +26,8 @@ def coefficient_of_performance(delta_T, source="air"): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_cop_profiles", simpl="", diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 951214bfc..cebea46b3 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -93,17 +93,20 @@ ----------- """ +import os import atlite import geopandas as gpd import pandas as pd -from _helpers import configure_logging, create_logger, mock_snakemake +from _helpers import configure_logging, create_logger logger = create_logger(__name__) if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("build_cutout", cutout="africa-2013-era5") configure_logging(snakemake) diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 83118d6c1..3686364b2 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -40,8 +40,8 @@ Then with a function that takes in the PyPSA network "base.nc", region and gadm shape data, the countries of interest, a scale factor, and the snapshots, it returns a csv file called "demand_profiles.csv", that allocates the load to the buses of the network according to GDP and population. """ - -import pathlib +import os +import os.path from itertools import product import geopandas as gpd @@ -54,9 +54,6 @@ BASE_DIR, configure_logging, create_logger, - get_path, - mock_snakemake, - normed, read_csv_nafix, read_osm_config, ) @@ -66,6 +63,10 @@ logger = create_logger(__name__) +def normed(s): + return s / s.sum() + + def get_gegis_regions(countries): """ Get the GEGIS region from the config file. @@ -112,8 +113,10 @@ def get_load_paths_gegis(ssp_parentfolder, config): prediction_year = config.get("load_options")["prediction_year"] ssp = config.get("load_options")["ssp"] + scenario_path = os.path.join(ssp_parentfolder, ssp) + load_paths = [] - load_dir = get_path( + load_dir = os.path.join( ssp_parentfolder, str(ssp), str(prediction_year), @@ -124,12 +127,12 @@ def get_load_paths_gegis(ssp_parentfolder, config): for continent in region_load: sel_ext = ".nc" for ext in [".nc", ".csv"]: - load_path = get_path(BASE_DIR, str(load_dir), str(continent) + str(ext)) - if get_path(load_path).exists(): + load_path = os.path.join(BASE_DIR, str(load_dir), str(continent) + str(ext)) + if os.path.exists(load_path): sel_ext = ext break file_name = str(continent) + str(sel_ext) - load_path = get_path(str(load_dir), file_name) + load_path = os.path.join(str(load_dir), file_name) load_paths.append(load_path) file_names.append(file_name) @@ -295,7 +298,10 @@ def upsample(cntry, group): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("build_demand_profiles") + configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) diff --git a/scripts/build_existing_heating_distribution.py b/scripts/build_existing_heating_distribution.py index 4391e9789..09d1cba8f 100644 --- a/scripts/build_existing_heating_distribution.py +++ b/scripts/build_existing_heating_distribution.py @@ -39,11 +39,11 @@ - "Mapping and analyses of the current and future (2020 - 2030) heating/cooling fuel deployment (fossil/renewables)" (https://energy.ec.europa.eu/publications/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment-fossilrenewables-1_en) """ import logging +import os import country_converter as coco import numpy as np import pandas as pd -from _helpers import mock_snakemake logger = logging.getLogger(__name__) @@ -165,6 +165,8 @@ def build_existing_heating(): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_existing_heating_distribution", simpl="", diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 17651bdab..685a595cc 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -6,15 +6,18 @@ Build heat demand time series. """ +import os + import atlite import geopandas as gpd import numpy as np import pandas as pd import xarray as xr -from _helpers import mock_snakemake if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("build_heat_demand", simpl="", clusters="10") time = pd.date_range(freq="h", **snakemake.params.snapshots) diff --git a/scripts/build_industrial_database.py b/scripts/build_industrial_database.py index f28c2f8ef..c565712eb 100644 --- a/scripts/build_industrial_database.py +++ b/scripts/build_industrial_database.py @@ -10,7 +10,7 @@ import pandas as pd import pycountry import requests -from _helpers import content_retrieve, mock_snakemake +from _helpers import content_retrieve from geopy.geocoders import Nominatim @@ -493,6 +493,8 @@ def create_paper_df(): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_industrial_database", simpl="", diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 6fbf66c39..83656b1d3 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -7,29 +7,21 @@ """ import logging +import os +import uuid from distutils.version import StrictVersion from itertools import product import geopandas as gpd import pandas as pd -from _helpers import locate_bus, mock_snakemake, three_2_two_digits_country +from _helpers import locate_bus, three_2_two_digits_country +from shapely.geometry import Point logger = logging.getLogger(__name__) gpd_version = StrictVersion(gpd.__version__) -def map_industry_to_buses( - df, - countries_list, - gadm_level_val, - geo_crs_val, - file_prefix_val, - gadm_url_prefix_val, - contended_flag_val, - gadm_input_file_args_list, - shapes_path_val, - gadm_clustering_val, -): +def map_industry_to_buses(df, countries, gadm_level, shapes_path, gadm_clustering): """ Load hotmaps database of industrial sites and map onto bus regions. Build industrial demand... Change name and add other functions. @@ -38,24 +30,19 @@ def map_industry_to_buses( Only cement not steel - proof of concept. Change hotmaps to more descriptive name, etc. """ - df = df[df.country.isin(countries_list)] + df = df[df.country.isin(countries)] df["gadm_{}".format(gadm_level)] = df[["x", "y", "country"]].apply( lambda site: locate_bus( site[["x", "y"]].astype("float"), site["country"], - gadm_level_val, - geo_crs_val, - file_prefix_val, - gadm_url_prefix_val, - gadm_input_file_args_list, - contended_flag_val, - path_to_gadm=shapes_path_val, - gadm_clustering=gadm_clustering_val, + gadm_level, + shapes_path, + gadm_clustering, ), axis=1, ) - return df.set_index("gadm_" + str(gadm_level_val)) + return df.set_index("gadm_" + str(gadm_level)) def build_nodal_distribution_key( @@ -131,6 +118,8 @@ def match_technology(df): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_industrial_distribution_key", simpl="", @@ -141,14 +130,12 @@ def match_technology(df): regions = gpd.read_file(snakemake.input.regions_onshore) shapes_path = snakemake.input.shapes_path + gadm_level = snakemake.params.gadm_level countries = snakemake.params.countries gadm_clustering = snakemake.params.alternative_clustering - geo_crs = snakemake.params.geo_crs - file_prefix = snakemake.params.gadm_file_prefix - gadm_url_prefix = snakemake.params.gadm_url_prefix - contended_flag = snakemake.params.contended_flag - gadm_input_file_args = ["data", "raw", "gadm"] + + # countries = ["EG", "BH"] if regions["name"][0][ :3 @@ -192,11 +179,6 @@ def match_technology(df): geo_locs[geo_locs.quality != "unavailable"], countries, gadm_level, - geo_crs, - file_prefix, - gadm_url_prefix, - contended_flag, - gadm_input_file_args, shapes_path, gadm_clustering, ) diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index ada38bdb9..8617fb466 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -9,10 +9,11 @@ """ import logging +import os from itertools import product import pandas as pd -from _helpers import BASE_DIR, get_path, mock_snakemake, read_csv_nafix +from _helpers import BASE_DIR, mock_snakemake, read_csv_nafix _logger = logging.getLogger(__name__) @@ -68,7 +69,7 @@ def country_to_nodal(industrial_production, keys): ) industry_demand = pd.read_csv( - get_path( + os.path.join( BASE_DIR, "data/custom/industry_demand_{0}_{1}.csv".format( snakemake.wildcards["demand"], @@ -208,7 +209,7 @@ def match_technology(df): aluminium_year = snakemake.params.aluminium_year AL = read_csv_nafix( - get_path(BASE_DIR, "data/AL_production.csv"), index_col=0 + os.path.join(BASE_DIR, "data/AL_production.csv"), index_col=0 ) # Filter data for the given year and countries AL_prod_tom = AL.query("Year == @aluminium_year and index in @countries_geo")[ diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index bc4233cef..efd6a4681 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -49,12 +49,10 @@ import atlite import geopandas as gpd import numpy as np -import pandas as pd import rasterio as rio -from _helpers import configure_logging, create_logger, get_path, mock_snakemake +from _helpers import configure_logging, create_logger from rasterio.features import geometry_mask from rasterio.warp import transform_bounds -from shapely.ops import unary_union logger = create_logger(__name__) @@ -67,14 +65,14 @@ def get_fileshapes(list_paths, accepted_formats=(".shp",)): list_fileshapes = [] for lf in list_paths: - if get_path(lf).is_dir(): # if it is a folder, then list all shapes files contained + if os.path.isdir(lf): # if it is a folder, then list all shapes files contained # loop over all dirs and subdirs for path, subdirs, files in os.walk(lf): # loop over all files for subfile in files: # add the subfile if it is a shape file if subfile.endswith(accepted_formats): - list_fileshapes.append(str(get_path(path, subfile))) + list_fileshapes.append(os.path.join(path, subfile)) elif lf.endswith(accepted_formats): list_fileshapes.append(lf) @@ -121,6 +119,9 @@ def unify_protected_shape_areas(inputs, natura_crs, out_logging): ------- unified_shape : GeoDataFrame with a unified "multishape" """ + import pandas as pd + from shapely.ops import unary_union + from shapely.validation import make_valid if out_logging: logger.info("Stage 3/5: Unify protected shape area.") @@ -175,6 +176,8 @@ def unify_protected_shape_areas(inputs, natura_crs, out_logging): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_natura_raster", cutouts=["cutouts/africa-2013-era5.nc"] ) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 1ebbd6ca6..867262abc 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -5,14 +5,14 @@ # -*- coding: utf-8 -*- +import os + import geopandas as gpd import numpy as np import pandas as pd from _helpers import ( - build_directory, configure_logging, create_logger, - mock_snakemake, read_geojson, read_osm_config, to_csv_nafix, @@ -42,7 +42,7 @@ def line_endings_to_bus_conversion(lines): # tol in m -def set_substations_ids(buses, distance_crs_val, tol=2000): +def set_substations_ids(buses, distance_crs, tol=2000): """ Function to set substations ids to buses, accounting for location tolerance. @@ -61,7 +61,7 @@ def set_substations_ids(buses, distance_crs_val, tol=2000): buses["station_id"] = -1 # create temporary series to execute distance calculations using m as reference distances - temp_bus_geom = buses.geometry.to_crs(distance_crs_val) + temp_bus_geom = buses.geometry.to_crs(distance_crs) # set tqdm options for substation ids tqdm_kwargs_substation_ids = dict( @@ -113,7 +113,7 @@ def set_substations_ids(buses, distance_crs_val, tol=2000): buses.loc[buses.index[close_nodes], "station_id"] = sub_id -def set_lines_ids(lines, buses, distance_crs_val): +def set_lines_ids(lines, buses, distance_crs): """ Function to set line buses ids to the closest bus in the list. """ @@ -129,8 +129,8 @@ def set_lines_ids(lines, buses, distance_crs_val): lines["bus0"] = -1 lines["bus1"] = -1 - busesepsg = buses.to_crs(distance_crs_val) - linesepsg = lines.to_crs(distance_crs_val) + busesepsg = buses.to_crs(distance_crs) + linesepsg = lines.to_crs(distance_crs) for i, row in tqdm(linesepsg.iterrows(), **tqdm_kwargs_line_ids): # select buses having the voltage level of the current line @@ -341,7 +341,7 @@ def get_transformers(buses, lines): return df_transformers -def get_converters(buses): +def get_converters(buses, lines): """ Function to create fake converter lines that connect buses of the same station_id of different polarities. @@ -407,6 +407,7 @@ def connect_stations_same_station_id(lines, buses): station_id_list = buses.station_id.unique() add_lines = [] + from shapely.geometry import LineString for s_id in station_id_list: buses_station_id = buses[buses.station_id == s_id] @@ -511,7 +512,7 @@ def set_lv_substations(buses): def merge_stations_lines_by_station_id_and_voltage( - lines, buses, distance_crs_val, tol=2000 + lines, buses, geo_crs, distance_crs, tol=2000 ): """ Function to merge close stations and adapt the line datasets to adhere to @@ -523,7 +524,7 @@ def merge_stations_lines_by_station_id_and_voltage( ) # set substation ids - set_substations_ids(buses, distance_crs_val, tol=tol) + set_substations_ids(buses, distance_crs, tol=tol) logger.info("Stage 3b/4: Merge substations with the same id") @@ -534,7 +535,7 @@ def merge_stations_lines_by_station_id_and_voltage( logger.info("Stage 3c/4: Specify the bus ids of the line endings") # set the bus ids to the line dataset - lines, buses = set_lines_ids(lines, buses, distance_crs_val) + lines, buses = set_lines_ids(lines, buses, distance_crs) # drop lines starting and ending in the same node lines.drop(lines[lines["bus0"] == lines["bus1"]].index, inplace=True) @@ -557,7 +558,9 @@ def merge_stations_lines_by_station_id_and_voltage( return lines, buses -def create_station_at_equal_bus_locations(lines, buses, distance_crs_val, tol=2000): +def create_station_at_equal_bus_locations( + lines, buses, geo_crs, distance_crs, tol=2000 +): # V1. Create station_id at same bus location # - We saw that buses are not connected exactly at one point, they are # usually connected to a substation "area" (analysed on maps) @@ -576,10 +579,10 @@ def create_station_at_equal_bus_locations(lines, buses, distance_crs_val, tol=20 bus_all = buses # set substation ids - set_substations_ids(buses, distance_crs_val, tol=tol) + set_substations_ids(buses, distance_crs, tol=tol) # set the bus ids to the line dataset - lines, buses = set_lines_ids(lines, buses, distance_crs_val) + lines, buses = set_lines_ids(lines, buses, distance_crs) # update line endings lines = line_endings_to_bus_conversion(lines) @@ -624,7 +627,7 @@ def _split_linestring_by_point(linestring, points): return list_linestrings -def fix_overpassing_lines(lines, buses, distance_crs_val, tol=1): +def fix_overpassing_lines(lines, buses, distance_crs, tol=1): """ Function to avoid buses overpassing lines with no connection when the bus is within a given tolerance from the line. @@ -635,8 +638,6 @@ def fix_overpassing_lines(lines, buses, distance_crs_val, tol=1): Geodataframe of lines buses : GeoDataFrame Geodataframe of substations - distance_crs_val: str - Coordinate reference system tol : float Tolerance in meters of the distance between the substation and the line below which the line will be split @@ -645,8 +646,8 @@ def fix_overpassing_lines(lines, buses, distance_crs_val, tol=1): lines_to_add = [] # list of lines to be added lines_to_split = [] # list of lines that have been split - lines_epsgmod = lines.to_crs(distance_crs_val) - buses_epsgmod = buses.to_crs(distance_crs_val) + lines_epsgmod = lines.to_crs(distance_crs) + buses_epsgmod = buses.to_crs(distance_crs) # set tqdm options for substation ids tqdm_kwargs_substation_ids = dict( @@ -709,7 +710,7 @@ def fix_overpassing_lines(lines, buses, distance_crs_val, tol=1): df_to_add.set_index(lines.index[-1] + df_to_add.index, inplace=True) # update length - df_to_add["length"] = df_to_add.to_crs(distance_crs_val).geometry.length + df_to_add["length"] = df_to_add.to_crs(distance_crs).geometry.length # update line endings df_to_add = line_endings_to_bus_conversion(df_to_add) @@ -737,13 +738,13 @@ def force_ac_lines(df, col="tag_frequency"): # TODO: default frequency may be by country default_ac_frequency = 50 - df[col] = default_ac_frequency + df["tag_frequency"] = default_ac_frequency df["dc"] = False return df -def add_buses_to_empty_countries(geo_crs_val, country_list, fp_country_shapes, buses): +def add_buses_to_empty_countries(country_list, fp_country_shapes, buses): """ Function to add a bus for countries missing substation data. """ @@ -761,7 +762,7 @@ def add_buses_to_empty_countries(geo_crs_val, country_list, fp_country_shapes, b no_data_countries_shape = ( country_shapes[country_shapes.index.isin(no_data_countries) == True] .reset_index() - .to_crs(geo_crs_val) + .to_crs(geo_crs) ) length = len(no_data_countries) df = gpd.GeoDataFrame( @@ -781,7 +782,7 @@ def add_buses_to_empty_countries(geo_crs_val, country_list, fp_country_shapes, b "geometry": no_data_countries_shape["geometry"].centroid, "substation_lv": [True] * length, }, - crs=geo_crs_val, + crs=geo_crs, ).astype( buses.dtypes.to_dict() ) # keep the same dtypes as buses @@ -810,8 +811,8 @@ def built_network( outputs, build_osm_network_config, countries_config, - geo_crs_val, - distance_crs_val, + geo_crs, + distance_crs, force_ac=False, ): logger.info("Stage 1/5: Read input data") @@ -843,14 +844,12 @@ def built_network( tol = build_osm_network_config.get("overpassing_lines_tolerance", 1) logger.info("Stage 3/5: Avoid nodes overpassing lines: enabled with tolerance") - lines, buses = fix_overpassing_lines(lines, buses, distance_crs_val, tol=tol) + lines, buses = fix_overpassing_lines(lines, buses, distance_crs, tol=tol) else: logger.info("Stage 3/5: Avoid nodes overpassing lines: disabled") # Add bus to countries with no buses - buses = add_buses_to_empty_countries( - geo_crs_val, countries_config, inputs.country_shapes, buses - ) + buses = add_buses_to_empty_countries(countries_config, inputs.country_shapes, buses) # METHOD to merge buses with same voltage and within tolerance Step 4/5 if build_osm_network_config.get("group_close_buses", False): @@ -859,7 +858,7 @@ def built_network( f"Stage 4/5: Aggregate close substations: enabled with tolerance {tol} m" ) lines, buses = merge_stations_lines_by_station_id_and_voltage( - lines, buses, distance_crs_val, tol=tol + lines, buses, geo_crs, distance_crs, tol=tol ) else: logger.info("Stage 4/5: Aggregate close substations: disabled") @@ -870,19 +869,21 @@ def built_network( transformers = get_transformers(buses, lines) # get converters: currently modelled as links connecting buses with different polarity - converters = get_converters(buses) + converters = get_converters(buses, lines) logger.info("Save outputs") # create clean directory if not already exist - build_directory(outputs["lines"]) + if not os.path.exists(outputs["lines"]): + os.makedirs(os.path.dirname(outputs["lines"]), exist_ok=True) to_csv_nafix(lines, outputs["lines"]) # Generate CSV to_csv_nafix(converters, outputs["converters"]) # Generate CSV to_csv_nafix(transformers, outputs["transformers"]) # Generate CSV # create clean directory if not already exist - build_directory(outputs["substations"]) + if not os.path.exists(outputs["substations"]): + os.makedirs(os.path.dirname(outputs["substations"]), exist_ok=True) # Generate CSV to_csv_nafix(buses, outputs["substations"]) @@ -891,6 +892,8 @@ def built_network( if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("build_osm_network") configure_logging(snakemake) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index ee950d4e8..5b2ea79ae 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -100,6 +100,8 @@ 4. OSM extraction was supposed to be ignoring non-generation features like CHP and Natural Gas storage (in contrast to PPM). """ +import os + import geopandas as gpd import numpy as np import pandas as pd @@ -109,10 +111,8 @@ from _helpers import ( configure_logging, create_logger, - get_current_directory_path, - get_path, - mock_snakemake, read_csv_nafix, + to_csv_nafix, two_digits_2_name_country, ) from scipy.spatial import cKDTree as KDTree @@ -121,62 +121,142 @@ logger = create_logger(__name__) -def add_power_plants( - custom_power_plants_file_path, - power_plants_config_dict, - ppl_assignment_strategy, - countries_names_list, -): +def convert_osm_to_pm(filepath_ppl_osm, filepath_ppl_pm): + if os.stat(filepath_ppl_osm).st_size == 0: + return to_csv_nafix(pd.DataFrame(), filepath_ppl_pm, index=False) + + add_ppls = read_csv_nafix(filepath_ppl_osm, index_col=0, dtype={"bus": "str"}) - if ppl_assignment_strategy == "replace": - # use only the data from custom_powerplants.csv - custom_power_plants = read_csv_nafix( - custom_power_plants_file_path, index_col=0, dtype={"bus": "str"} + custom_ppls_coords = gpd.GeoSeries.from_wkt(add_ppls["geometry"]) + add_ppls = ( + add_ppls.rename( + columns={ + "name": "Name", + "tags.generator:source": "Fueltype", + "tags.generator:type": "Technology", + "tags.power": "Set", + "power_output_MW": "Capacity", + } ) - return custom_power_plants - elif ppl_assignment_strategy == "merge": - # merge the data from powerplantmatching and custom_powerplants.csv - ppl_ppm = ( - pm.powerplants( - from_url=False, update=True, config_update=power_plants_config_dict - ) - .powerplant.fill_missing_decommissioning_years() - .query( - 'Fueltype not in ["Solar", "Wind"] and Country in @countries_names_list' + .replace( + dict( + Fueltype={ + "nuclear": "Nuclear", + "wind": "Wind", + "hydro": "Hydro", + "tidal": "Other", + "wave": "Other", + "geothermal": "Geothermal", + "solar": "Solar", + # "Hard Coal" follows defaults of PPM + "coal": "Hard Coal", + "gas": "Natural Gas", + "biomass": "Bioenergy", + "biofuel": "Bioenergy", + "biogas": "Bioenergy", + "oil": "Oil", + "diesel": "Oil", + "gasoline": "Oil", + "waste": "Waste", + "osmotic": "Other", + "wave": "Other", + # approximation + # TODO: this shall be improved, one entry shall be Oil and the otherone gas + "gas;oil": "Oil", + "steam": "Natural Gas", + "waste_heat": "Other", + }, + Technology={ + "combined_cycle": "CCGT", + "gas_turbine": "OCGT", + "steam_turbine": "Steam Turbine", + "reciprocating_engine": "Combustion Engine", + # a very strong assumption + "wind_turbine": "Onshore", + "horizontal_axis": "Onshore", + "vertical_axis": "Offhore", + "solar_photovoltaic_panel": "Pv", + }, + Set={"generator": "PP", "plant": "PP"}, ) - .powerplant.convert_country_to_alpha2() - .pipe(replace_natural_gas_technology) ) - ppl_cpp = read_csv_nafix( - custom_power_plants_file_path, index_col=0, dtype={"bus": "str"} - ) - power_plants = pd.concat( - [ppl_ppm, ppl_cpp], sort=False, ignore_index=True, verify_integrity=True - ) - return power_plants - elif ( - ppl_assignment_strategy not in ["merge", "replace"] - or ppl_assignment_strategy is None - ): - # use only the data from powerplantsmatching - power_plants = ( - pm.powerplants( - from_url=False, update=True, config_update=power_plants_config_dict - ) - .powerplant.fill_missing_decommissioning_years() - .query( - 'Fueltype not in ["Solar", "Wind"] and Country in @countries_names_list' - ) - .powerplant.convert_country_to_alpha2() - .pipe(replace_natural_gas_technology) + .assign( + Country=lambda df: df.Country.map(two_digits_2_name_country), + # Name=lambda df: "OSM_" + # + df.Country.astype(str) + # + "_" + # + df.id.astype(str) + # + "-" + # + df.Name.astype(str), + Efficiency="", + Duration="", + Volume_Mm3="", + DamHeight_m="", + StorageCapacity_MWh="", + DateIn="", + DateRetrofit="", + DateMothball="", + DateOut="", + lat=custom_ppls_coords.y, + lon=custom_ppls_coords.x, + EIC=lambda df: df.id, + projectID=lambda df: "OSM" + df.id.astype(str), ) - return power_plants - else: - raise Exception( - "No power plants were built for custom_powerplants {}".format( - ppl_assignment_strategy - ) + .dropna(subset=["Fueltype"]) + ) + + # All Hydro objects can be interpreted by PPM as Storages, too + # However, everything extracted from OSM seems to belong + # to power plants with "tags.power" == "generator" only + osm_ppm_df = pd.DataFrame( + data={ + "osm_method": ["run-of-the-river", "water-pumped-storage", "water-storage"], + "ppm_technology": ["Run-Of-River", "Pumped Storage", "Reservoir"], + } + ) + for i in osm_ppm_df.index: + add_ppls.loc[ + add_ppls["tags.generator:method"] == osm_ppm_df.loc[i, "osm_method"], + "Technology", + ] = osm_ppm_df.loc[i, "ppm_technology"] + + # originates from osm::"tags.generator:source" + add_ppls.loc[add_ppls["Fueltype"] == "Nuclear", "Technology"] = "Steam Turbine" + + # PMM contains data on NG, batteries and hydro storages + # trying to catch some of them... + # originates from osm::"tags.generator:source" + add_ppls.loc[add_ppls["Fueltype"] == "battery", "Set"] = "Store" + # originates from osm::tags.generator:type + add_ppls.loc[add_ppls["Technology"] == "battery storage", "Set"] = "Store" + + add_ppls = add_ppls.replace(dict(Fueltype={"battery": "Other"})).drop( + columns=["tags.generator:method", "geometry", "Area", "id"], + errors="ignore", + ) + + to_csv_nafix(add_ppls, filepath_ppl_pm, index=False) + + return add_ppls + + +def add_custom_powerplants(ppl, inputs, config): + if "custom_powerplants" not in config["electricity"]: + return ppl + + custom_ppl_query = config["electricity"]["custom_powerplants"] + if not custom_ppl_query: + return ppl + add_ppls = read_csv_nafix( + inputs.custom_powerplants, index_col=0, dtype={"bus": "str"} + ) + + if custom_ppl_query == "merge": + return pd.concat( + [ppl, add_ppls], sort=False, ignore_index=True, verify_integrity=True ) + elif custom_ppl_query == "replace": + return add_ppls def replace_natural_gas_technology(df: pd.DataFrame): @@ -214,36 +294,36 @@ def replace_natural_gas_technology(df: pd.DataFrame): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("build_powerplants") configure_logging(snakemake) with open(snakemake.input.pm_config, "r") as f: - powerplants_config = yaml.safe_load(f) + config = yaml.safe_load(f) filepath_osm_ppl = snakemake.input.osm_powerplants filepath_osm2pm_ppl = snakemake.output.powerplants_osm2pm - powerplants_assignment_strategy = snakemake.params.custom_powerplants n = pypsa.Network(snakemake.input.base_network) countries_codes = n.buses.country.unique() countries_names = list(map(two_digits_2_name_country, countries_codes)) - powerplants_config["target_countries"] = countries_names + config["target_countries"] = countries_names if ( "EXTERNAL_DATABASE" - in powerplants_config["matching_sources"] - + powerplants_config["fully_included_sources"] + in config["matching_sources"] + config["fully_included_sources"] ): - if "EXTERNAL_DATABASE" not in powerplants_config: + if "EXTERNAL_DATABASE" not in config: logger.error( "Missing configuration EXTERNAL_DATABASE in powerplantmatching config yaml\n\t" "Please check file configs/powerplantmatching_config.yaml" ) logger.info("Parsing OSM generator data to powerplantmatching format") - powerplants_config["EXTERNAL_DATABASE"]["fn"] = get_path( - get_current_directory_path(), filepath_osm2pm_ppl + config["EXTERNAL_DATABASE"]["fn"] = os.path.join( + os.getcwd(), filepath_osm2pm_ppl ) else: # create an empty file @@ -253,20 +333,23 @@ def replace_natural_gas_technology(df: pd.DataFrame): # specify the main query for filtering powerplants ppl_query = snakemake.params.powerplants_filter if isinstance(ppl_query, str): - powerplants_config["main_query"] = ppl_query + config["main_query"] = ppl_query else: - powerplants_config["main_query"] = "" - - ppl = add_power_plants( - snakemake.input.custom_powerplants_file, - powerplants_config, - powerplants_assignment_strategy, - countries_names, + config["main_query"] = "" + + ppl = ( + pm.powerplants(from_url=False, update=True, config_update=config) + .powerplant.fill_missing_decommissioning_years() + .query('Fueltype not in ["Solar", "Wind"] and Country in @countries_names') + .powerplant.convert_country_to_alpha2() + .pipe(replace_natural_gas_technology) ) - countries_without_ppl = [ - c for c in countries_codes if c not in ppl.Country.unique() - ] + ppl = add_custom_powerplants( + ppl, snakemake.input, snakemake.config + ) # add carriers from own powerplant files + + cntries_without_ppl = [c for c in countries_codes if c not in ppl.Country.unique()] for c in countries_codes: substation_i = n.buses.query("substation_lv and country == @c").index @@ -276,8 +359,8 @@ def replace_natural_gas_technology(df: pd.DataFrame): tree_i = kdtree.query(ppl.loc[ppl_i, ["lon", "lat"]].values)[1] ppl.loc[ppl_i, "bus"] = substation_i.append(pd.Index([np.nan]))[tree_i] - if countries_without_ppl: - logger.warning(f"No powerplants known in: {', '.join(countries_without_ppl)}") + if cntries_without_ppl: + logger.warning(f"No powerplants known in: {', '.join(cntries_without_ppl)}") bus_null_b = ppl["bus"].isnull() if bus_null_b.any(): diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 76bacd687..8d3722de6 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -6,10 +6,12 @@ # -*- coding: utf-8 -*- import multiprocessing as mp +import os import shutil from itertools import takewhile from operator import attrgetter +import fiona import geopandas as gpd import numpy as np import pandas as pd @@ -17,15 +19,12 @@ import requests import xarray as xr from _helpers import ( - build_directory, BASE_DIR, configure_logging, create_logger, - get_gadm_layer, - get_path, - mock_snakemake, three_2_two_digits_country, two_2_three_digits_country, + two_digits_2_name_country, ) from numba import njit from numba.core import types @@ -40,6 +39,210 @@ logger = create_logger(__name__) +def get_GADM_filename(country_code): + """ + Function to get the GADM filename given the country code. + """ + special_codes_GADM = { + "XK": "XKO", # kosovo + "CP": "XCL", # clipperton island + "SX": "MAF", # sint maartin + "TF": "ATF", # french southern territories + "AX": "ALA", # aland + "IO": "IOT", # british indian ocean territory + "CC": "CCK", # cocos island + "NF": "NFK", # norfolk + "PN": "PCN", # pitcairn islands + "JE": "JEY", # jersey + "XS": "XSP", # spratly + "GG": "GGY", # guernsey + "UM": "UMI", # united states minor outlying islands + "SJ": "SJM", # svalbard + "CX": "CXR", # Christmas island + } + + if country_code in special_codes_GADM: + return f"gadm41_{special_codes_GADM[country_code]}" + else: + return f"gadm41_{two_2_three_digits_country(country_code)}" + + +def download_GADM(country_code, update=False, out_logging=False): + """ + Download gpkg file from GADM for a given country code. + + Parameters + ---------- + country_code : str + Two letter country codes of the downloaded files + update : bool + Update = true, forces re-download of files + + Returns + ------- + gpkg file per country + """ + GADM_filename = get_GADM_filename(country_code) + GADM_url = f"https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/{GADM_filename}.gpkg" + + GADM_inputfile_gpkg = os.path.join( + BASE_DIR, + "data", + "gadm", + GADM_filename, + GADM_filename + ".gpkg", + ) # Input filepath gpkg + + if not os.path.exists(GADM_inputfile_gpkg) or update is True: + if out_logging: + logger.warning( + f"Stage 5 of 5: {GADM_filename} of country {two_digits_2_name_country(country_code)} does not exist, downloading to {GADM_inputfile_gpkg}" + ) + # create data/osm directory + os.makedirs(os.path.dirname(GADM_inputfile_gpkg), exist_ok=True) + + try: + r = requests.get(GADM_url, stream=True, timeout=300) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + raise Exception( + f"GADM server is down at {GADM_url}. Data needed for building shapes can't be extracted.\n\r" + ) + except Exception as exception: + raise Exception( + f"An error happened when trying to load GADM data by {GADM_url}.\n\r" + + str(exception) + + "\n\r" + ) + else: + with open(GADM_inputfile_gpkg, "wb") as f: + shutil.copyfileobj(r.raw, f) + + return GADM_inputfile_gpkg, GADM_filename + + +def filter_gadm( + geodf, + layer, + cc, + contended_flag, + output_nonstd_to_csv=False, +): + # identify non standard geodf rows + geodf_non_std = geodf[geodf["GID_0"] != two_2_three_digits_country(cc)].copy() + + if not geodf_non_std.empty: + logger.info( + f"Contended areas have been found for gadm layer {layer}. They will be treated according to {contended_flag} option" + ) + + # NOTE: in these options GID_0 is not changed because it is modified below + if contended_flag == "drop": + geodf.drop(geodf_non_std.index, inplace=True) + elif contended_flag != "set_by_country": + # "set_by_country" option is the default; if this elif applies, the desired option falls back to the default + logger.warning( + f"Value '{contended_flag}' for option contented_flag is not recognized.\n" + + "Fallback to 'set_by_country'" + ) + + # force GID_0 to be the country code for the relevant countries + geodf["GID_0"] = cc + + # country shape should have a single geometry + if (layer == 0) and (geodf.shape[0] > 1): + logger.warning( + f"Country shape is composed by multiple shapes that are being merged in agreement to contented_flag option '{contended_flag}'" + ) + # take the first row only to re-define geometry keeping other columns + geodf = geodf.iloc[[0]].set_geometry([geodf.unary_union]) + + # debug output to file + if output_nonstd_to_csv and not geodf_non_std.empty: + geodf_non_std.to_csv( + f"resources/non_standard_gadm{layer}_{cc}_raw.csv", index=False + ) + + return geodf + + +def get_GADM_layer( + country_list, + layer_id, + geo_crs, + contended_flag, + update=False, + outlogging=False, +): + """ + Function to retrieve a specific layer id of a geopackage for a selection of + countries. + + Parameters + ---------- + country_list : str + List of the countries + layer_id : int + Layer to consider in the format GID_{layer_id}. + When the requested layer_id is greater than the last available layer, then the last layer is selected. + When a negative value is requested, then, the last layer is requested + """ + # initialization of the geoDataFrame + geodf_list = [] + + for country_code in country_list: + # Set the current layer id (cur_layer_id) to global layer_id + cur_layer_id = layer_id + + # download file gpkg + file_gpkg, name_file = download_GADM(country_code, update, outlogging) + + # get layers of a geopackage + list_layers = fiona.listlayers(file_gpkg) + + # get layer name + if (cur_layer_id < 0) or (cur_layer_id >= len(list_layers)): + # when layer id is negative or larger than the number of layers, select the last layer + cur_layer_id = len(list_layers) - 1 + + # read gpkg file + geodf_temp = gpd.read_file( + file_gpkg, layer="ADM_ADM_" + str(cur_layer_id), engine="pyogrio" + ).to_crs(geo_crs) + + geodf_temp = filter_gadm( + geodf=geodf_temp, + layer=cur_layer_id, + cc=country_code, + contended_flag=contended_flag, + output_nonstd_to_csv=False, + ) + + # create a subindex column that is useful + # in the GADM processing of sub-national zones + geodf_temp["GADM_ID"] = geodf_temp[f"GID_{cur_layer_id}"] + + # from pypsa-earth-sec + # if layer_id == 0: + # geodf_temp["GADM_ID"] = geodf_temp[f"GID_{cur_layer_id}"].apply( + # lambda x: two_2_three_digits_country(x[:2]) + # ) + pd.Series(range(1, geodf_temp.shape[0] + 1)).astype(str) + # else: + # # create a subindex column that is useful + # # in the GADM processing of sub-national zones + # # Fix issues with missing "." in selected cases + # geodf_temp["GADM_ID"] = geodf_temp[f"GID_{cur_layer_id}"].apply( + # lambda x: x if x[3] == "." else x[:3] + "." + x[3:] + # ) + + # append geodataframes + geodf_list.append(geodf_temp) + + geodf_GADM = gpd.GeoDataFrame(pd.concat(geodf_list, ignore_index=True)) + geodf_GADM.set_crs(geo_crs) + + return geodf_GADM + + def _simplify_polys(polys, minarea=0.01, tolerance=0.01, filterremote=False): "Function to simplify the shape polygons" if isinstance(polys, MultiPolygon): @@ -59,29 +262,17 @@ def _simplify_polys(polys, minarea=0.01, tolerance=0.01, filterremote=False): return polys.simplify(tolerance=tolerance) -def get_countries_shapes( - countries, - geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, - contended_flag, - update, - out_logging, -): +def countries(countries, geo_crs, contended_flag, update=False, out_logging=False): "Create country shapes" if out_logging: logger.info("Stage 1 of 5: Create country shapes") # download data if needed and get the layer id 0, corresponding to the countries - df_countries = get_gadm_layer( + df_countries = get_GADM_layer( countries, 0, geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, contended_flag, update, out_logging, @@ -115,7 +306,8 @@ def country_cover(country_shapes, eez_shapes=None, out_logging=False, distance=0 def save_to_geojson(df, fn): - get_path(fn).unlink(missing_ok=True) # remove file if it exists + if os.path.exists(fn): + os.unlink(fn) # remove file if it exists if not isinstance(df, gpd.GeoDataFrame): df = gpd.GeoDataFrame(dict(geometry=df)) @@ -130,19 +322,19 @@ def save_to_geojson(df, fn): pass -def load_eez(countries_codes, geo_crs, eez_gpkg_file="./data/eez/eez_v11.gpkg"): +def load_EEZ(countries_codes, geo_crs, EEZ_gpkg="./data/eez/eez_v11.gpkg"): """ Function to load the database of the Exclusive Economic Zones. The dataset shall be downloaded independently by the user (see guide) or together with pypsa-earth package. """ - if not get_path(eez_gpkg_file).exists(): + if not os.path.exists(EEZ_gpkg): raise Exception( - f"File EEZ {eez_gpkg_file} not found, please download it from https://www.marineregions.org/download_file.php?name=World_EEZ_v11_20191118_gpkg.zip and copy it in {get_path(eez_gpkg).parent}" + f"File EEZ {EEZ_gpkg} not found, please download it from https://www.marineregions.org/download_file.php?name=World_EEZ_v11_20191118_gpkg.zip and copy it in {os.path.dirname(EEZ_gpkg)}" ) - geodf_EEZ = gpd.read_file(eez_gpkg_file, engine="pyogrio").to_crs(geo_crs) + geodf_EEZ = gpd.read_file(EEZ_gpkg, engine="pyogrio").to_crs(geo_crs) geodf_EEZ.dropna(axis=0, how="any", subset=["ISO_TER1"], inplace=True) # [["ISO_TER1", "TERRITORY1", "ISO_SOV1", "ISO_SOV2", "ISO_SOV3", "geometry"]] geodf_EEZ = geodf_EEZ[["ISO_TER1", "geometry"]] @@ -162,11 +354,11 @@ def load_eez(countries_codes, geo_crs, eez_gpkg_file="./data/eez/eez_v11.gpkg"): return geodf_EEZ -def get_eez( +def eez( countries, geo_crs, country_shapes, - eez_gpkg_file, + EEZ_gpkg, out_logging=False, distance=0.01, minarea=0.01, @@ -182,7 +374,7 @@ def get_eez( logger.info("Stage 2 of 5: Create offshore shapes") # load data - df_eez = load_eez(countries, geo_crs, eez_gpkg_file) + df_eez = load_EEZ(countries, geo_crs, EEZ_gpkg) eez_countries = [cc for cc in countries if df_eez.name.str.contains(cc).any()] ret_df = gpd.GeoDataFrame( @@ -235,7 +427,7 @@ def download_WorldPop( worldpop_method: str worldpop_method = "api" will use the API method to access the WorldPop 100mx100m dataset. worldpop_method = "standard" will use the standard method to access the WorldPop 1KMx1KM dataset. country_code : str - Two-letter country codes of the downloaded files. + Two letter country codes of the downloaded files. Files downloaded from https://data.worldpop.org/ datasets WorldPop UN adjusted year : int Year of the data to download @@ -245,7 +437,7 @@ def download_WorldPop( Minimum size of each file to download """ if worldpop_method == "api": - return download_WorldPop_API(country_code, year, update) + return download_WorldPop_API(country_code, year, update, out_logging, size_min) elif worldpop_method == "standard": return download_WorldPop_standard( @@ -267,7 +459,7 @@ def download_WorldPop_standard( Parameters ---------- country_code : str - Two-letter country codes of the downloaded files. + Two letter country codes of the downloaded files. Files downloaded from https://data.worldpop.org/ datasets WorldPop UN adjusted year : int Year of the data to download @@ -297,17 +489,17 @@ def download_WorldPop_standard( f"https://data.worldpop.org/GIS/Population/Global_2000_2020_Constrained/2020/maxar_v1/{two_2_three_digits_country(country_code).upper()}/{WorldPop_filename}", ] - WorldPop_inputfile = get_path( + WorldPop_inputfile = os.path.join( BASE_DIR, "data", "WorldPop", WorldPop_filename ) # Input filepath tif - if not get_path(WorldPop_inputfile).exists() or update is True: + if not os.path.exists(WorldPop_inputfile) or update is True: if out_logging: logger.warning( f"Stage 3 of 5: {WorldPop_filename} does not exist, downloading to {WorldPop_inputfile}" ) # create data/osm directory - build_directory(WorldPop_inputfile) + os.makedirs(os.path.dirname(WorldPop_inputfile), exist_ok=True) loaded = False for WorldPop_url in WorldPop_urls: @@ -323,7 +515,9 @@ def download_WorldPop_standard( return WorldPop_inputfile, WorldPop_filename -def download_WorldPop_API(country_code, year=2020, size_min=300): +def download_WorldPop_API( + country_code, year=2020, update=False, out_logging=False, size_min=300 +): """ Download tiff file for each country code using the api method from worldpop API with 100mx100m resolution. @@ -331,10 +525,14 @@ def download_WorldPop_API(country_code, year=2020, size_min=300): Parameters ---------- country_code : str - Two-letter country codes of the downloaded files. + Two letter country codes of the downloaded files. Files downloaded from https://data.worldpop.org/ datasets WorldPop UN adjusted year : int Year of the data to download + update : bool + Update = true, forces re-download of files + size_min : int + Minimum size of each file to download Returns ------- WorldPop_inputfile : str @@ -345,10 +543,10 @@ def download_WorldPop_API(country_code, year=2020, size_min=300): WorldPop_filename = f"{two_2_three_digits_country(country_code).lower()}_ppp_{year}_UNadj_constrained.tif" # Request to get the file - WorldPop_inputfile = get_path( + WorldPop_inputfile = os.path.join( BASE_DIR, "data", "WorldPop", WorldPop_filename ) # Input filepath tif - build_directory(WorldPop_inputfile) + os.makedirs(os.path.dirname(WorldPop_inputfile), exist_ok=True) year_api = int(str(year)[2:]) loaded = False WorldPop_api_urls = [ @@ -370,7 +568,7 @@ def download_WorldPop_API(country_code, year=2020, size_min=300): return WorldPop_inputfile, WorldPop_filename -def convert_gdp(name_file_nc, year=2015, out_logging=False): +def convert_GDP(name_file_nc, year=2015, out_logging=False): """ Function to convert the nc database of the GDP to tif, based on the work at https://doi.org/10.1038/sdata.2018.4. The dataset shall be downloaded independently by the user (see guide) or together with pypsa-earth package. @@ -383,19 +581,15 @@ def convert_gdp(name_file_nc, year=2015, out_logging=False): name_file_tif = name_file_nc[:-2] + "tif" # path of the nc file - GDP_nc = get_path( - BASE_DIR, "data", "GDP", name_file_nc - ) # Input filepath nc + GDP_nc = os.path.join(BASE_DIR, "data", "GDP", name_file_nc) # Input filepath nc # path of the tif file - GDP_tif = get_path( - BASE_DIR, "data", "GDP", name_file_tif - ) # Input filepath nc + GDP_tif = os.path.join(BASE_DIR, "data", "GDP", name_file_tif) # Input filepath nc # Check if file exists, otherwise throw exception - if not get_path(GDP_nc).exists(): + if not os.path.exists(GDP_nc): raise Exception( - f"File {name_file_nc} not found, please download it from https://datadryad.org/stash/dataset/doi:10.5061/dryad.dk1j0 and copy it in {get_path(GDP_nc).parent}" + f"File {name_file_nc} not found, please download it from https://datadryad.org/stash/dataset/doi:10.5061/dryad.dk1j0 and copy it in {os.path.dirname(GDP_nc)}" ) # open nc dataset @@ -417,7 +611,7 @@ def convert_gdp(name_file_nc, year=2015, out_logging=False): return GDP_tif, name_file_tif -def load_gdp( +def load_GDP( year=2015, update=False, out_logging=False, @@ -433,16 +627,14 @@ def load_gdp( # path of the nc file name_file_tif = name_file_nc[:-2] + "tif" - GDP_tif = get_path( - BASE_DIR, "data", "GDP", name_file_tif - ) # Input filepath tif + GDP_tif = os.path.join(BASE_DIR, "data", "GDP", name_file_tif) # Input filepath tif - if update | (not get_path(GDP_tif).exists()): + if update | (not os.path.exists(GDP_tif)): if out_logging: logger.warning( f"Stage 5 of 5: File {name_file_tif} not found, the file will be produced by processing {name_file_nc}" ) - convert_gdp(name_file_nc, year, out_logging) + convert_GDP(name_file_nc, year, out_logging) return GDP_tif, name_file_tif @@ -483,6 +675,8 @@ def add_gdp_data( update=False, out_logging=False, name_file_nc="GDP_PPP_1990_2015_5arcmin_v2.nc", + nprocesses=2, + disable_progressbar=False, ): """ Function to add gdp data to arbitrary number of shapes in a country. @@ -505,7 +699,7 @@ def add_gdp_data( # initialize new gdp column df_gadm["gdp"] = 0.0 - GDP_tif, name_tif = load_gdp(year, update, out_logging, name_file_nc) + GDP_tif, name_tif = load_GDP(year, update, out_logging, name_file_nc) with rasterio.open(GDP_tif) as src: # resample data to target shape @@ -931,6 +1125,7 @@ def add_population_data( out_logging=False, mem_read_limit_per_process=1024, nprocesses=2, + disable_progressbar=False, ): """ Function to add population data to arbitrary number of shapes in a country. @@ -1046,14 +1241,11 @@ def add_population_data( pbar.update(1) -def get_gadm_shapes( +def gadm( worldpop_method, gdp_method, countries, geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, contended_flag, mem_mb, layer_id=2, @@ -1066,17 +1258,7 @@ def get_gadm_shapes( logger.info("Stage 3 of 5: Creation GADM GeoDataFrame") # download data if needed and get the desired layer_id - df_gadm = get_gadm_layer( - countries, - layer_id, - geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, - contended_flag, - update, - out_logging, - ) + df_gadm = get_GADM_layer(countries, layer_id, geo_crs, contended_flag, update) # select and rename columns df_gadm.rename(columns={"GID_0": "country"}, inplace=True) @@ -1113,7 +1295,7 @@ def get_gadm_shapes( name_file_nc="GDP_PPP_1990_2015_5arcmin_v2.nc", ) - # renaming three-letter to two-letter ISO code before saving GADM file + # renaming 3 letter to 2 letter ISO code before saving GADM file # In the case of a contested territory in the form 'Z00.00_0', save 'AA.00_0' # Include bugfix for the case of 'XXX00_0' where the "." is missing, such as for Ghana df_gadm["GADM_ID"] = df_gadm["country"] + df_gadm["GADM_ID"].str[3:].apply( @@ -1131,12 +1313,14 @@ def get_gadm_shapes( if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("build_shapes") configure_logging(snakemake) out = snakemake.output - eez_gpkg = snakemake.input["eez"] + EEZ_gpkg = snakemake.input["eez"] mem_mb = snakemake.resources["mem_mb"] countries_list = snakemake.params.countries @@ -1151,41 +1335,32 @@ def get_gadm_shapes( contended_flag = snakemake.params.build_shape_options["contended_flag"] worldpop_method = snakemake.params.build_shape_options["worldpop_method"] gdp_method = snakemake.params.build_shape_options["gdp_method"] - file_prefix = snakemake.params.build_shape_options["gadm_file_prefix"] - gadm_url_prefix = snakemake.params.build_shape_options["gadm_url_prefix"] - gadm_input_file_args = ["data", "gadm"] - country_shapes_df = get_countries_shapes( + country_shapes = countries( countries_list, geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, contended_flag, update, out_logging, ) - country_shapes_df.to_file(snakemake.output.country_shapes) + country_shapes.to_file(snakemake.output.country_shapes) - offshore_shapes = get_eez( - countries_list, geo_crs, country_shapes_df, eez_gpkg, out_logging + offshore_shapes = eez( + countries_list, geo_crs, country_shapes, EEZ_gpkg, out_logging ) offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) africa_shape = gpd.GeoDataFrame( - geometry=[country_cover(country_shapes_df, offshore_shapes.geometry)] + geometry=[country_cover(country_shapes, offshore_shapes.geometry)] ) africa_shape.reset_index().to_file(snakemake.output.africa_shape) - gadm_shapes = get_gadm_shapes( + gadm_shapes = gadm( worldpop_method, gdp_method, countries_list, geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, contended_flag, mem_mb, layer_id, diff --git a/scripts/build_ship_profile.py b/scripts/build_ship_profile.py index 4ffa7837e..f3f8465e8 100644 --- a/scripts/build_ship_profile.py +++ b/scripts/build_ship_profile.py @@ -4,10 +4,11 @@ # SPDX-License-Identifier: AGPL-3.0-or-later import logging +import os +from pathlib import Path import numpy as np import pandas as pd -from _helpers import mock_snakemake logger = logging.getLogger(__name__) @@ -63,6 +64,9 @@ def build_ship_profile(export_volume, ship_opts): if __name__ == "__main__": if "snakemake" not in globals(): + + from _helpers import mock_snakemake + snakemake = mock_snakemake( "build_ship_profile", h2export="120", diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 6ffd6be74..ec5dbb2fe 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -6,15 +6,17 @@ Build solar thermal collector time series. """ +import os + import atlite import geopandas as gpd import numpy as np import pandas as pd import xarray as xr -from _helpers import mock_snakemake if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_solar_thermal_profiles", diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index bfa2ad598..bd7de5156 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -5,15 +5,17 @@ """ Build temperature profiles. """ +import os + import atlite import geopandas as gpd import numpy as np import pandas as pd import xarray as xr -from _helpers import mock_snakemake if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake snakemake = mock_snakemake( "build_temperature_profiles", diff --git a/scripts/build_test_configs.py b/scripts/build_test_configs.py index e64a6d407..1f0cb00c5 100644 --- a/scripts/build_test_configs.py +++ b/scripts/build_test_configs.py @@ -14,8 +14,9 @@ """ import collections.abc import copy +import os +from pathlib import Path -from _helpers import get_current_directory_path, get_path, mock_snakemake from ruamel.yaml import YAML @@ -36,7 +37,7 @@ def _parse_inputconfig(input_config, yaml): return input_config if isinstance(input_config, str): - input_config = get_path(get_current_directory_path(), input_config) + input_config = Path(Path.cwd(), input_config) with open(input_config) as fp: return yaml.load(fp) @@ -75,7 +76,7 @@ def create_test_config(default_config, diff_config, output_path): # Output path if isinstance(output_path, str): - output_path = get_path(get_current_directory_path(), output_path) + output_path = Path(Path.cwd(), output_path) # Save file yaml.dump(merged_config, output_path) @@ -85,6 +86,8 @@ def create_test_config(default_config, diff_config, output_path): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("build_test_configs") # Input paths diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py index dbab231b3..5362e3c21 100644 --- a/scripts/clean_osm_data.py +++ b/scripts/clean_osm_data.py @@ -5,6 +5,8 @@ # -*- coding: utf-8 -*- +import os + import geopandas as gpd import numpy as np import pandas as pd @@ -13,8 +15,6 @@ REGION_COLS, configure_logging, create_logger, - get_path_size, - mock_snakemake, save_to_geojson, to_csv_nafix, ) @@ -900,7 +900,7 @@ def clean_data( ): logger.info("Process OSM lines") - if get_path_size(input_files["lines"]) > 0: + if os.path.getsize(input_files["lines"]) > 0: # Load raw data lines df_lines = load_network_data("lines", data_options) @@ -915,7 +915,7 @@ def clean_data( df_all_lines = df_lines # load cables only if data are stored - if get_path_size(input_files["cables"]) > 0: + if os.path.getsize(input_files["cables"]) > 0: logger.info("Add OSM cables to data") # Load raw data lines df_cables = load_network_data("cables", data_options) @@ -965,7 +965,7 @@ def clean_data( logger.info("Process OSM substations") - if get_path_size(input_files["substations"]) > 0: + if os.path.getsize(input_files["substations"]) > 0: df_all_substations = load_network_data("substations", data_options) # prepare dataset for substations @@ -1025,7 +1025,7 @@ def clean_data( logger.info("Process OSM generators") - if get_path_size(input_files["generators"]) > 0: + if os.path.getsize(input_files["generators"]) > 0: df_all_generators = gpd.read_file(input_files["generators"]) # prepare the generator dataset @@ -1060,8 +1060,9 @@ def clean_data( if __name__ == "__main__": if "snakemake" not in globals(): - snakemake = mock_snakemake("clean_osm_data") + from _helpers import mock_snakemake + snakemake = mock_snakemake("clean_osm_data") configure_logging(snakemake) tag_substation = snakemake.params.clean_osm_data_options["tag_substation"] diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 551776e36..eeaa2a98a 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -121,6 +121,7 @@ :align: center """ +import os from functools import reduce import geopandas as gpd @@ -133,9 +134,6 @@ configure_logging, create_logger, get_aggregation_strategies, - get_path, - mock_snakemake, - normed, update_p_nom_max, ) from add_electricity import load_costs @@ -146,7 +144,6 @@ busmap_by_kmeans, get_clustering_from_busmap, ) -from scipy.sparse import csgraph from shapely.geometry import Point idx = pd.IndexSlice @@ -154,6 +151,10 @@ logger = create_logger(__name__) +def normed(x): + return (x / x.sum()).fillna(0.0) + + def weighting_for_country(n, x): conv_carriers = {"OCGT", "CCGT", "PHS", "hydro"} gen = n.generators.loc[n.generators.carrier.isin(conv_carriers)].groupby( @@ -377,7 +378,7 @@ def n_bounds(model, *n_id): ) -def busmap_for_gadm_clusters(inputs, n, gadm_level): +def busmap_for_gadm_clusters(inputs, n, gadm_level, geo_crs, country_list): gdf = gpd.read_file(inputs.gadm_shapes) def locate_bus(coords, co): @@ -425,6 +426,7 @@ def busmap_for_n_clusters( algorithm_kwds.setdefault("random_state", 0) def fix_country_assignment_for_hac(n): + from scipy.sparse import csgraph # overwrite country of nodes that are disconnected from their country-topology for country in n.buses.country.unique(): @@ -559,6 +561,7 @@ def clustering_for_n_clusters( n_clusters, alternative_clustering, gadm_layer_id, + geo_crs, country_list, distribution_cluster, build_shape_options, @@ -578,7 +581,9 @@ def clustering_for_n_clusters( if not isinstance(custom_busmap, pd.Series): if alternative_clustering: - busmap = busmap_for_gadm_clusters(inputs, n, gadm_layer_id) + busmap = busmap_for_gadm_clusters( + inputs, n, gadm_layer_id, geo_crs, country_list + ) else: busmap = busmap_for_n_clusters( inputs, @@ -626,7 +631,8 @@ def clustering_for_n_clusters( def save_to_geojson(s, fn): - get_path(fn).unlink(missing_ok=True) + if os.path.exists(fn): + os.unlink(fn) df = s.reset_index() schema = {**gpd.io.file.infer_schema(df), "geometry": "Unknown"} df.to_file(fn, driver="GeoJSON", schema=schema) @@ -648,6 +654,8 @@ def cluster_regions(busmaps, inputs, output): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "cluster_network", network="elec", simpl="", clusters="min" ) @@ -662,6 +670,7 @@ def cluster_regions(busmaps, inputs, output): gadm_layer_id = snakemake.params.build_shape_options["gadm_layer_id"] focus_weights = snakemake.params.get("focus_weights", None) country_list = snakemake.params.countries + geo_crs = snakemake.params.geo_crs renewable_carriers = pd.Index( [ @@ -743,6 +752,7 @@ def consense(x): n_clusters, alternative_clustering, gadm_layer_id, + geo_crs, country_list, distribution_cluster, snakemake.params.build_shape_options, diff --git a/scripts/copy_config.py b/scripts/copy_config.py index e40d48700..b7073c9fa 100644 --- a/scripts/copy_config.py +++ b/scripts/copy_config.py @@ -2,20 +2,24 @@ # SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors # # SPDX-License-Identifier: AGPL-3.0-or-later - +import os from shutil import copy -from _helpers import BASE_DIR, get_path, mock_snakemake +from _helpers import BASE_DIR files_to_copy = { - get_path(BASE_DIR, "./config.yaml"): "config.yaml", - get_path(BASE_DIR, "./Snakefile"): "Snakefile", - get_path(BASE_DIR, "./scripts/solve_network.py"): "solve_network.py", - get_path(BASE_DIR, "./scripts/prepare_sector_network.py"): "prepare_sector_network.py", + os.path.join(BASE_DIR, "./config.yaml"): "config.yaml", + os.path.join(BASE_DIR, "./Snakefile"): "Snakefile", + os.path.join(BASE_DIR, "./scripts/solve_network.py"): "solve_network.py", + os.path.join( + BASE_DIR, "./scripts/prepare_sector_network.py" + ): "prepare_sector_network.py", } if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("copy_config") directory = snakemake.output["folder"] diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index b36472087..c327a7ae4 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -26,16 +26,11 @@ - ``data/osm/out``: Prepared power data as .geojson and .csv files per country - ``resources/osm/raw``: Prepared and per type (e.g. cable/lines) aggregated power data as .geojson and .csv files """ +import os import shutil +from pathlib import Path -from _helpers import ( - BASE_DIR, - configure_logging, - create_logger, - get_path, - mock_snakemake, - read_osm_config, -) +from _helpers import BASE_DIR, configure_logging, create_logger, read_osm_config from earth_osm import eo logger = create_logger(__name__) @@ -97,15 +92,17 @@ def convert_iso_to_geofk( if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("download_osm_data") configure_logging(snakemake) run = snakemake.config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" - store_path_resources = get_path( - BASE_DIR, "resources", RDIR, "osm", "raw" + store_path_resources = Path.joinpath( + Path(BASE_DIR), "resources", RDIR, "osm", "raw" ) - store_path_data = get_path(BASE_DIR, "data", "osm") + store_path_data = Path.joinpath(Path(BASE_DIR), "data", "osm") country_list = country_list_to_geofk(snakemake.params.countries) eo.save_osm_data( @@ -121,9 +118,10 @@ def convert_iso_to_geofk( progress_bar=snakemake.config["enable"]["progress_bar"], ) - out_path = get_path(store_path_resources, "out") + out_path = Path.joinpath(store_path_resources, "out") names = ["generator", "cable", "line", "substation"] out_formats = ["csv", "geojson"] + new_files = os.listdir(out_path) # list downloaded osm files # earth-osm (eo) only outputs files with content # If the file is empty, it is not created @@ -132,9 +130,9 @@ def convert_iso_to_geofk( # Rename and move osm files to the resources folder output for name in names: for f in out_formats: - new_file_name = get_path(store_path_resources, f"all_raw_{name}s.{f}") - old_files = list(get_path(out_path).glob(f"*{name}.{f}")) - # if file is missing, create empty file, otherwise rename it and move it + new_file_name = Path.joinpath(store_path_resources, f"all_raw_{name}s.{f}") + old_files = list(Path(out_path).glob(f"*{name}.{f}")) + # if file is missing, create empty file, otherwise rename it an move it if not old_files: with open(new_file_name, "w") as f: pass diff --git a/scripts/make_statistics.py b/scripts/make_statistics.py index e71070329..2b84e48fc 100644 --- a/scripts/make_statistics.py +++ b/scripts/make_statistics.py @@ -23,21 +23,15 @@ ------- This rule creates a dataframe containing in the columns the relevant statistics for the current run. """ +import os +from pathlib import Path import geopandas as gpd import numpy as np import pandas as pd import pypsa import xarray as xr -from _helpers import ( - create_country_list, - create_logger, - get_path, - get_path_size, - mock_snakemake, - three_2_two_digits_country, - to_csv_nafix, -) +from _helpers import create_logger, mock_snakemake, to_csv_nafix from build_test_configs import create_test_config from shapely.validation import make_valid @@ -49,7 +43,9 @@ def _multi_index_scen(rulename, keys): def _mock_snakemake(rule, **kwargs): + snakemake = mock_snakemake(rule, **kwargs) + return snakemake @@ -75,6 +71,7 @@ def generate_scenario_by_country( out_dir : str (optional) Output directory where output configuration files are executed """ + from _helpers import create_country_list, three_2_two_digits_country clean_country_list = create_country_list(country_list) @@ -126,7 +123,7 @@ def collect_basic_osm_stats(path, rulename, header): """ Collect basic statistics on OSM data: number of items """ - if get_path(path).is_file() and get_path_size(path) > 0: + if Path(path).is_file() and Path(path).stat().st_size > 0: df = gpd.read_file(path) n_elem = len(df) @@ -145,7 +142,7 @@ def collect_network_osm_stats(path, rulename, header, metric_crs="EPSG:3857"): - length of the stored shapes - length of objects with tag_frequency == 0 (DC elements) """ - if get_path(path).is_file() and get_path_size(path) > 0: + if Path(path).is_file() and Path(path).stat().st_size > 0: df = gpd.read_file(path) n_elem = len(df) obj_length = ( @@ -247,7 +244,7 @@ def collect_bus_regions_stats(bus_region_rule="build_bus_regions"): df = pd.DataFrame() - if get_path(fp_onshore).is_file() and get_path(fp_offshore).is_file(): + if Path(fp_onshore).is_file() and Path(fp_offshore).is_file(): gdf_onshore = gpd.read_file(fp_onshore) gdf_offshore = gpd.read_file(fp_offshore) @@ -289,7 +286,7 @@ def capacity_stats(df): else: return df.groupby("carrier").p_nom.sum().astype(float) - if get_path(network_path).is_file(): + if Path(network_path).is_file(): n = pypsa.Network(network_path) lines_length = float((n.lines.length * n.lines.num_parallel).sum()) @@ -344,7 +341,7 @@ def collect_shape_stats(rulename="build_shapes", area_crs="ESRI:54009"): """ snakemake = _mock_snakemake(rulename) - if not get_path(snakemake.output.africa_shape).is_file(): + if not Path(snakemake.output.africa_shape).is_file(): return pd.DataFrame() df_continent = gpd.read_file(snakemake.output.africa_shape) @@ -355,7 +352,7 @@ def collect_shape_stats(rulename="build_shapes", area_crs="ESRI:54009"): .geometry.area.iloc[0] ) - if not get_path(snakemake.output.gadm_shapes).is_file(): + if not Path(snakemake.output.gadm_shapes).is_file(): return pd.DataFrame() df_gadm = gpd.read_file(snakemake.output.gadm_shapes) @@ -469,7 +466,7 @@ def collect_renewable_stats(rulename, technology): """ snakemake = _mock_snakemake(rulename, technology=technology) - if get_path(snakemake.output.profile).is_file(): + if Path(snakemake.output.profile).is_file(): res = xr.open_dataset(snakemake.output.profile) if technology == "hydro": @@ -502,7 +499,7 @@ def add_computational_stats(df, snakemake, column_name=None): comp_data = [np.nan] * 3 # total_time, mean_load and max_memory if snakemake.benchmark: - if not get_path(snakemake.benchmark).is_file(): + if not Path(snakemake.benchmark).is_file(): return df bench_data = pd.read_csv(snakemake.benchmark, delimiter="\t") @@ -582,6 +579,8 @@ def calculate_stats( if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("make_statistics") fp_stats = snakemake.output["stats"] diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 26b66157d..ccddcef6a 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -51,17 +51,12 @@ Replacing *summaries* with *plots* creates nice colored maps of the results. """ +import os import pandas as pd import pypsa -from _helpers import ( - build_directory, - configure_logging, - create_logger, - get_path, - mock_snakemake, -) -from add_electricity import load_costs, update_transmission_costs +from _helpers import configure_logging +from add_electricity import create_logger, load_costs, update_transmission_costs idx = pd.IndexSlice @@ -501,7 +496,7 @@ def make_summaries(networks_dict, inputs, cost_config, elec_config, country="all for label, filename in networks_dict.items(): print(label, filename) - if not get_path(filename).exists(): + if not os.path.exists(filename): print("does not exist!!") continue @@ -532,13 +527,15 @@ def make_summaries(networks_dict, inputs, cost_config, elec_config, country="all def to_csv(dfs, dir): - build_directory(dir, just_parent_directory=False) + os.makedirs(dir, exist_ok=True) for key, df in dfs.items(): - df.to_csv(get_path(dir, f"{key}.csv")) + df.to_csv(os.path.join(dir, f"{key}.csv")) if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "make_summary", simpl="", @@ -553,9 +550,9 @@ def to_csv(dfs, dir): scenario_name = snakemake.config.get("run", {}).get("name", "") if scenario_name: - network_dir = get_path(network_dir, "results", scenario_name, "networks") + network_dir = os.path.join(network_dir, "results", scenario_name, "networks") else: - network_dir = get_path(network_dir, "results", "networks") + network_dir = os.path.join(network_dir, "results", "networks") configure_logging(snakemake) @@ -571,7 +568,7 @@ def expand_from_wildcard(key): ll = [snakemake.wildcards.ll] networks_dict = { - (simpl, clusters, l, opts): get_path( + (simpl, clusters, l, opts): os.path.join( network_dir, f"elec_s{simpl}_" f"{clusters}_ec_l{l}_{opts}.nc" ) for simpl in expand_from_wildcard("simpl") diff --git a/scripts/monte_carlo.py b/scripts/monte_carlo.py index 726f0ed78..b8d0ab1dd 100644 --- a/scripts/monte_carlo.py +++ b/scripts/monte_carlo.py @@ -17,7 +17,7 @@ add_to_snakefile: false # When set to true, enables Monte Carlo sampling samples: 9 # number of optimizations. Note that number of samples when using scipy has to be the square of a prime number sampling_strategy: "chaospy" # "pydoe2", "chaospy", "scipy", packages that are supported - seed: 42 # set seedling for reproducibility + seed: 42 # set seedling for reproducibilty uncertainties: loads_t.p_set: type: uniform @@ -67,6 +67,8 @@ wildcard {unc}, which is described in the config.yaml and created in the Snakefile as a range from 0 to (total number of) SAMPLES. """ +import os + import chaospy import numpy as np import pandas as pd @@ -75,7 +77,7 @@ from _helpers import configure_logging, create_logger from pyDOE2 import lhs from scipy.stats import beta, gamma, lognorm, norm, qmc, triang -from sklearn.preprocessing import MinMaxScaler, minmax_scale +from sklearn.preprocessing import MinMaxScaler from solve_network import * logger = create_logger(__name__) @@ -98,6 +100,8 @@ def monte_carlo_sampling_pydoe2( Adapted from Disspaset: https://github.com/energy-modelling-toolkit/Dispa-SET/blob/master/scripts/build_and_run_hypercube.py Documentation on PyDOE2: https://github.com/clicumu/pyDOE2 (fixes latin_cube errors) """ + from pyDOE2 import lhs + from scipy.stats import qmc # Generate a Nfeatures-dimensional latin hypercube varying between 0 and 1: lh = lhs( @@ -112,7 +116,7 @@ def monte_carlo_sampling_pydoe2( lh = rescale_distribution(lh, uncertainties_values) discrepancy = qmc.discrepancy(lh) logger.info( - f"Discrepancy is: {discrepancy} more details in function documentation." + "Discrepancy is:", discrepancy, " more details in function documentation." ) return lh @@ -131,6 +135,8 @@ def monte_carlo_sampling_chaospy( Documentation on Chaospy: https://github.com/clicumu/pyDOE2 (fixes latin_cube errors) Documentation on Chaospy latin-hyper cube (quasi-Monte Carlo method): https://chaospy.readthedocs.io/en/master/user_guide/fundamentals/quasi_random_samples.html#Quasi-random-samples """ + import chaospy + from scipy.stats import qmc # generate a Nfeatures-dimensional latin hypercube varying between 0 and 1: N_FEATURES = "chaospy.Uniform(0, 1), " * N_FEATURES @@ -142,7 +148,7 @@ def monte_carlo_sampling_chaospy( lh = rescale_distribution(lh, uncertainties_values) discrepancy = qmc.discrepancy(lh) logger.info( - f"Discrepancy is: {discrepancy} more details in function documentation." + "Discrepancy is:", discrepancy, " more details in function documentation." ) return lh @@ -172,6 +178,7 @@ def monte_carlo_sampling_scipy( Documentation for Latin Hypercube: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.qmc.LatinHypercube.html#scipy.stats.qmc.LatinHypercube Orthogonal LHS is better than basic LHS: https://github.com/scipy/scipy/pull/14546/files, https://en.wikipedia.org/wiki/Latin_hypercube_sampling """ + from scipy.stats import qmc sampler = qmc.LatinHypercube( d=N_FEATURES, @@ -185,7 +192,7 @@ def monte_carlo_sampling_scipy( lh = rescale_distribution(lh, uncertainties_values) discrepancy = qmc.discrepancy(lh) logger.info( - f"Discrepancy is: {discrepancy} more details in function documentation." + "Discrepancy is:", discrepancy, " more details in function documentation." ) return lh @@ -225,37 +232,36 @@ def rescale_distribution( - The function supports rescaling for uniform, normal, lognormal, triangle, beta, and gamma distributions. - The rescaled samples will have values in the range [0, 1]. """ + from scipy.stats import beta, gamma, lognorm, norm, qmc, triang + from sklearn.preprocessing import MinMaxScaler, minmax_scale for idx, value in enumerate(uncertainties_values): dist = value.get("type") params = value.get("args") - if dist == "uniform": - l_bounds, u_bounds = params - latin_hypercube[:, idx] = minmax_scale( - latin_hypercube[:, idx], feature_range=(l_bounds, u_bounds) - ) - elif dist == "normal": - mean, std = params - latin_hypercube[:, idx] = norm.ppf(latin_hypercube[:, idx], mean, std) - elif dist == "lognormal": - shape = params[0] - latin_hypercube[:, idx] = lognorm.ppf(latin_hypercube[:, idx], s=shape) - elif dist == "triangle": - mid_point = params[0] - latin_hypercube[:, idx] = triang.ppf(latin_hypercube[:, idx], mid_point) - elif dist == "beta": - a, b = params - latin_hypercube[:, idx] = beta.ppf(latin_hypercube[:, idx], a, b) - elif dist == "gamma": - shape, scale = params - latin_hypercube[:, idx] = gamma.ppf(latin_hypercube[:, idx], shape, scale) - else: - exception_message = ( - f"The value {dist} is not among the allowed ones: uniform, normal, lognormal, " - f"triangle, beta, gamma" - ) - raise NotImplementedError(exception_message) + match dist: + case "uniform": + l_bounds, u_bounds = params + latin_hypercube[:, idx] = minmax_scale( + latin_hypercube[:, idx], feature_range=(l_bounds, u_bounds) + ) + case "normal": + mean, std = params + latin_hypercube[:, idx] = norm.ppf(latin_hypercube[:, idx], mean, std) + case "lognormal": + shape = params[0] + latin_hypercube[:, idx] = lognorm.ppf(latin_hypercube[:, idx], s=shape) + case "triangle": + mid_point = params[0] + latin_hypercube[:, idx] = triang.ppf(latin_hypercube[:, idx], mid_point) + case "beta": + a, b = params + latin_hypercube[:, idx] = beta.ppf(latin_hypercube[:, idx], a, b) + case "gamma": + shape, scale = params + latin_hypercube[:, idx] = gamma.ppf( + latin_hypercube[:, idx], shape, scale + ) # samples space needs to be from 0 to 1 mm = MinMaxScaler(feature_range=(0, 1), clip=True) @@ -342,6 +348,8 @@ def validate_parameters( if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "monte_carlo", simpl="", diff --git a/scripts/non_workflow/zenodo_handler.py b/scripts/non_workflow/zenodo_handler.py index 41d31ccbc..ab65fd7d7 100644 --- a/scripts/non_workflow/zenodo_handler.py +++ b/scripts/non_workflow/zenodo_handler.py @@ -18,7 +18,7 @@ Relevant Settings ----------------- """ -import pathlib +from pathlib import Path import zenodopy @@ -65,7 +65,7 @@ metadata=METADATA, ) for path in UPLOAD_PATHS: - path = pathlib.Path.joinpath(pathlib.Path(ROOT), path) + path = Path.joinpath(Path(ROOT), path) zeno.upload_zip(source_dir=str(path)) @@ -76,7 +76,7 @@ metadata=METADATA, ) for path in UPLOAD_PATHS: - path = pathlib.Path.joinpath(pathlib.Path(ROOT), path) + path = Path.joinpath(Path(ROOT), path) if path.exists(): if TYPE == "upload": if path.exists(): diff --git a/scripts/non_workflow/zip_folder.py b/scripts/non_workflow/zip_folder.py index 4931d3bde..020ff0780 100644 --- a/scripts/non_workflow/zip_folder.py +++ b/scripts/non_workflow/zip_folder.py @@ -8,8 +8,9 @@ Module to zip the desired folders to be stored in google drive, or equivalent. """ import os -import pathlib import zipfile +from os.path import basename +from xml.etree.ElementInclude import include # Zip the files from given directory that matches the filter @@ -22,7 +23,7 @@ def zipFilesInDir(dirName, zipFileName, filter, include_parent=True): for filename in filenames: if filter(filename): # create complete filepath of file in directory - filePath = str(pathlib.Path(folderName, filename)) + filePath = os.path.join(folderName, filename) # path of the zip file if include_parent: diff --git a/scripts/override_respot.py b/scripts/override_respot.py index 665af0654..b6d78d02b 100644 --- a/scripts/override_respot.py +++ b/scripts/override_respot.py @@ -3,8 +3,15 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later +import os +from itertools import dropwhile +from types import SimpleNamespace + +import numpy as np import pandas as pd import pypsa +import pytz +import xarray as xr from _helpers import mock_snakemake, override_component_attrs diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 1dc782cbd..3bcac8f52 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -17,6 +17,8 @@ ----------- """ +import os + import cartopy.crs as ccrs import geopandas as gpd import matplotlib as mpl @@ -30,7 +32,6 @@ configure_logging, create_logger, load_network_for_plots, - mock_snakemake, ) from matplotlib.legend_handler import HandlerPatch from matplotlib.patches import Circle, Ellipse @@ -42,14 +43,14 @@ def assign_location(n): for c in n.iterate_components(n.one_port_components | n.branch_components): - i_find = pd.Series(c.df.index.str.find(" ", start=4), c.df.index) + ifind = pd.Series(c.df.index.str.find(" ", start=4), c.df.index) - for i in i_find.value_counts().index: + for i in ifind.value_counts().index: # these have already been assigned defaults if i == -1: continue - names = i_find.index[i_find == i] + names = ifind.index[ifind == i] c.df.loc[names, "location"] = names.str[:i] @@ -1068,6 +1069,10 @@ def plot_sector_map( if __name__ == "__main__": if "snakemake" not in globals(): + import os + + from _helpers import mock_snakemake + snakemake = mock_snakemake( "plot_network", network="elec", diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index a2618bdc1..77d1217e6 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -16,10 +16,11 @@ Description ----------- """ +import os import matplotlib.pyplot as plt import pandas as pd -from _helpers import configure_logging, create_logger, get_path, mock_snakemake +from _helpers import configure_logging, create_logger logger = create_logger(__name__) @@ -216,6 +217,7 @@ def plot_energy(infn, snmk, fn=None): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake snakemake = mock_snakemake( "plot_summary", @@ -238,7 +240,7 @@ def plot_energy(infn, snmk, fn=None): logger.error(f"plotting function for {summary} has not been defined") func( - get_path(snakemake.input[0], f"{summary}.csv"), + os.path.join(snakemake.input[0], f"{summary}.csv"), snakemake, snakemake.output[0], ) diff --git a/scripts/prepare_airports.py b/scripts/prepare_airports.py index 79cb91018..e69b20438 100644 --- a/scripts/prepare_airports.py +++ b/scripts/prepare_airports.py @@ -3,8 +3,11 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later +import numpy as np import pandas as pd -from _helpers import mock_snakemake + +# from _helpers import configure_logging + # logger = logging.getLogger(__name__) @@ -35,7 +38,10 @@ def download_airports(): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("prepare_airports") + # configure_logging(snakemake) # run = snakemake.config.get("run", {}) # RDIR = run["name"] + "/" if run.get("name") else "" diff --git a/scripts/prepare_db.py b/scripts/prepare_db.py index 21a4122cd..88da74e7d 100644 --- a/scripts/prepare_db.py +++ b/scripts/prepare_db.py @@ -16,10 +16,12 @@ @author: haz43975 """ +import os + import matplotlib.pyplot as plt +import numpy as np import pandas as pd import pypsa -from _helpers import mock_snakemake # %% @@ -29,6 +31,7 @@ if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake snakemake = mock_snakemake( "prepare_db", diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py index 1f616c187..708420b27 100644 --- a/scripts/prepare_energy_totals.py +++ b/scripts/prepare_energy_totals.py @@ -2,8 +2,22 @@ # SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors # # SPDX-License-Identifier: AGPL-3.0-or-later +import glob import logging -from _helpers import BASE_DIR, get_path, mock_snakemake, read_csv_nafix +import os +import sys +from io import BytesIO +from pathlib import Path +from urllib.request import urlopen +from zipfile import ZipFile + +import country_converter as coco +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import py7zr +import requests +from _helpers import BASE_DIR, read_csv_nafix, three_2_two_digits_country _logger = logging.getLogger(__name__) @@ -24,6 +38,8 @@ def calculate_end_values(df): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "prepare_energy_totals", simpl="", @@ -37,7 +53,9 @@ def calculate_end_values(df): investment_year = int(snakemake.wildcards.planning_horizons) demand_sc = snakemake.wildcards.demand # loading the demand scenrario wildcard - base_energy_totals = read_csv_nafix(get_path(BASE_DIR, "data", "energy_totals_base.csv"), index_col=0) + base_energy_totals = read_csv_nafix( + os.path.join(BASE_DIR, "data/energy_totals_base.csv"), index_col=0 + ) growth_factors_cagr = read_csv_nafix( snakemake.input.growth_factors_cagr, index_col=0 ) diff --git a/scripts/prepare_heat_data.py b/scripts/prepare_heat_data.py index d50e1ea72..54c9dd959 100644 --- a/scripts/prepare_heat_data.py +++ b/scripts/prepare_heat_data.py @@ -2,8 +2,10 @@ # SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors # # SPDX-License-Identifier: AGPL-3.0-or-later +import os from itertools import product +import numpy as np import pandas as pd import pypsa import pytz @@ -130,6 +132,8 @@ def prepare_heat_data(n): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "prepare_heat_data", simpl="", diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 7996d2b7c..47d847b78 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -56,7 +56,7 @@ for all ``scenario`` s in the configuration file the rule :mod:`prepare_network`. """ -import pathlib +import os import re from zipfile import ZipFile @@ -65,15 +65,7 @@ import pandas as pd import pypsa import requests -from _helpers import ( - BASE_DIR, - configure_logging, - create_logger, - get_current_directory_path, - get_path, - mock_snakemake, - two_2_three_digits_country, -) +from _helpers import BASE_DIR, configure_logging, create_logger from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice @@ -93,18 +85,19 @@ def download_emission_data(): try: url = "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/EDGAR/datasets/v60_GHG/CO2_excl_short-cycle_org_C/v60_GHG_CO2_excl_short-cycle_org_C_1970_2018.zip" with requests.get(url) as rq: - with open(get_path(BASE_DIR, "data", "co2.zip"), "wb") as file: + with open(os.path.join(BASE_DIR, "data/co2.zip"), "wb") as file: file.write(rq.content) - file_path = get_path(BASE_DIR, "data", "co2.zip") + file_path = os.path.join(BASE_DIR, "data/co2.zip") with ZipFile(file_path, "r") as zipObj: - zipObj.extract("v60_CO2_excl_short-cycle_org_C_1970_2018.xls", get_path(BASE_DIR, "data")) - pathlib.Path(file_path).unlink(missing_ok=True) + zipObj.extract( + "v60_CO2_excl_short-cycle_org_C_1970_2018.xls", + os.path.join(BASE_DIR, "data"), + ) + os.remove(file_path) return "v60_CO2_excl_short-cycle_org_C_1970_2018.xls" - except requests.exceptions.RequestException as e: - logger.error( - f"Failed download resource from '{url}' with exception message '{e}'." - ) - raise SystemExit(e) + except: + logger.error(f"Failed download resource from '{url}'.") + return False def emission_extractor(filename, emission_year, country_names): @@ -119,7 +112,7 @@ def emission_extractor(filename, emission_year, country_names): emission_year : int Year of CO2 emissions country_names : numpy.ndarray - Two-letter country codes of analysed countries. + Two letter country codes of analysed countries. Returns ------- @@ -127,8 +120,8 @@ def emission_extractor(filename, emission_year, country_names): """ # data reading process - data_path = get_path(BASE_DIR, "data", filename) - df = pd.read_excel(data_path, sheet_name="v6.0_EM_CO2_fossil_IPCC1996", skiprows=8) + datapath = os.path.join(BASE_DIR, "data", filename) + df = pd.read_excel(datapath, sheet_name="v6.0_EM_CO2_fossil_IPCC1996", skiprows=8) df.columns = df.iloc[0] df = df.set_index("Country_code_A3") df = df.loc[ @@ -136,10 +129,9 @@ def emission_extractor(filename, emission_year, country_names): ] df = df.loc[:, "Y_1970":"Y_2018"].astype(float).ffill(axis=1) df = df.loc[:, "Y_1970":"Y_2018"].astype(float).bfill(axis=1) - cc_iso3 = [ - two_2_three_digits_country(two_code_country) - for two_code_country in country_names - ] + cc_iso3 = cc.convert(names=country_names, to="ISO3") + if len(country_names) == 1: + cc_iso3 = [cc_iso3] emission_by_country = df.loc[ df.index.intersection(cc_iso3), "Y_" + str(emission_year) ] @@ -192,7 +184,7 @@ def set_line_s_max_pu(n, s_max_pu): logger.info(f"N-1 security margin of lines set to {s_max_pu}") -def set_transmission_limit(n, ll_type, factor, costs): +def set_transmission_limit(n, ll_type, factor, costs, Nyears=1): links_dc_b = n.links.carrier == "DC" if not n.links.empty else pd.Series() _lines_s_nom = ( @@ -325,6 +317,8 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "prepare_network", simpl="", @@ -332,6 +326,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): ll="v0.3", opts="Co2L-24H", ) + configure_logging(snakemake) opts = snakemake.wildcards.opts.split("-") @@ -365,7 +360,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if snakemake.params.electricity["automatic_emission"]: - country_names = n.buses.country.unique().tolist() + country_names = n.buses.country.unique() emission_year = snakemake.params.electricity[ "automatic_emission_base_year" ] @@ -429,7 +424,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] - set_transmission_limit(n, ll_type, factor, costs) + set_transmission_limit(n, ll_type, factor, costs, Nyears) set_line_nom_max( n, diff --git a/scripts/prepare_transport_data.py b/scripts/prepare_transport_data.py index 315de4dec..48e3bbcf7 100644 --- a/scripts/prepare_transport_data.py +++ b/scripts/prepare_transport_data.py @@ -2,13 +2,13 @@ # SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors # # SPDX-License-Identifier: AGPL-3.0-or-later +import os import numpy as np import pandas as pd import pypsa import pytz import xarray as xr -from _helpers import mock_snakemake def transport_degree_factor( @@ -93,6 +93,7 @@ def prepare_transport_data(n): weekly_profile=traffic.values, ) + nodal_transport_shape = transport_shape / transport_shape.sum().sum() transport_shape = transport_shape / transport_shape.sum() transport_data = pd.read_csv( @@ -203,6 +204,8 @@ def prepare_transport_data(n): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "prepare_transport_data", simpl="", diff --git a/scripts/prepare_transport_data_input.py b/scripts/prepare_transport_data_input.py index 2f821d14d..d932d1174 100644 --- a/scripts/prepare_transport_data_input.py +++ b/scripts/prepare_transport_data_input.py @@ -2,14 +2,17 @@ # SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors # # SPDX-License-Identifier: AGPL-3.0-or-later +import logging +import os import shutil +from pathlib import Path import country_converter as coco import numpy as np import pandas as pd -from _helpers import BASE_DIR, configure_logging, create_logger, get_path, mock_snakemake +from _helpers import BASE_DIR -logger = create_logger(__name__) +# logger = logging.getLogger(__name__) def download_number_of_vehicles(): @@ -28,9 +31,9 @@ def download_number_of_vehicles(): Nbr_vehicles_csv = pd.read_csv( fn, storage_options=storage_options, encoding="utf8" ) - logger.info("File at {} read successfully.".format(fn)) + print("File read successfully.") except Exception as e: - logger.error("Failed to read the file from {} with exception:".format(fn), e) + print("Failed to read the file:", e) return pd.DataFrame() Nbr_vehicles_csv = Nbr_vehicles_csv.rename( @@ -106,10 +109,16 @@ def download_CO2_emissions(): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake snakemake = mock_snakemake("prepare_transport_data_input") - configure_logging(snakemake) + # configure_logging(snakemake) + + # run = snakemake.config.get("run", {}) + # RDIR = run["name"] + "/" if run.get("name") else "" + # store_path_data = Path.joinpath(Path().cwd(), "data") + # country_list = country_list_to_geofk(snakemake.config["countries"])' # Downloaded and prepare vehicles_csv: vehicles_csv = download_number_of_vehicles().copy() diff --git a/scripts/prepare_urban_percent.py b/scripts/prepare_urban_percent.py index df5bb46df..65e683dac 100644 --- a/scripts/prepare_urban_percent.py +++ b/scripts/prepare_urban_percent.py @@ -2,12 +2,17 @@ # SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors # # SPDX-License-Identifier: AGPL-3.0-or-later +import os import country_converter as coco import pandas as pd import py7zr import requests -from _helpers import get_path, mock_snakemake + +# from _helpers import configure_logging + + +# logger = logging.getLogger(__name__) def download_urban_percent(): @@ -46,14 +51,16 @@ def download_urban_percent(): print(f"Urban percent extracted successfully") # Read the extracted CSV file - csv_filename = get_path(filename).stem # Remove the .7z extension to get the CSV filename + csv_filename = os.path.splitext(filename)[ + 0 + ] # Remove the .7z extension to get the CSV filename urban_percent_orig = pd.read_csv(csv_filename) print("Urban percent CSV file read successfully:") # Remove the downloaded .7z and .csv files - get_path(filename).unlink(missing_ok=True) - get_path(csv_filename).unlink(missing_ok=True) + os.remove(filename) + os.remove(csv_filename) else: print(f"Failed to download file: Status code {response.status_code}") @@ -63,6 +70,8 @@ def download_urban_percent(): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("prepare_urban_percent") df = download_urban_percent().copy() diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index 7ecdc158a..297599d4a 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -81,6 +81,7 @@ """ import datetime as dt +import os import re from zipfile import ZipFile @@ -92,8 +93,6 @@ configure_logging, create_country_list, create_logger, - get_path, - mock_snakemake, progress_retrieve, ) from google_drive_downloader import GoogleDriveDownloader as gdd @@ -120,9 +119,9 @@ def load_databundle_config(config): return config -def download_and_unzip_zenodo(config, root_path, hot_run=True, disable_progress=False): +def download_and_unzip_zenodo(config, rootpath, hot_run=True, disable_progress=False): """ - download_and_unzip_zenodo(config, root_path, dest_path, hot_run=True, + download_and_unzip_zenodo(config, rootpath, dest_path, hot_run=True, disable_progress=False) Function to download and unzip the data from zenodo @@ -131,7 +130,7 @@ def download_and_unzip_zenodo(config, root_path, hot_run=True, disable_progress= ------ config : Dict Configuration data for the category to download - root_path : str + rootpath : str Absolute path of the repository hot_run : Bool (default True) When true the data are downloaded @@ -144,8 +143,8 @@ def download_and_unzip_zenodo(config, root_path, hot_run=True, disable_progress= True when download is successful, False otherwise """ resource = config["category"] - file_path = get_path(root_path, "tempfile.zip") - destination = get_path(BASE_DIR, config["destination"]) + file_path = os.path.join(rootpath, "tempfile.zip") + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["zenodo"] if hot_run: @@ -156,20 +155,18 @@ def download_and_unzip_zenodo(config, root_path, hot_run=True, disable_progress= with ZipFile(file_path, "r") as zipObj: # Extract all the contents of zip file in current directory zipObj.extractall(path=destination) - get_path(file_path).unlink(missing_ok=True) + os.remove(file_path) logger.info(f"Downloaded resource '{resource}' from cloud '{url}'.") - except Exception as e: - logger.warning( - f"Failed download resource '{resource}' from cloud '{url}' with exception message '{e}'." - ) + except: + logger.warning(f"Failed download resource '{resource}' from cloud '{url}'.") return False return True -def download_and_unzip_gdrive(config, root_path, hot_run=True, disable_progress=False): +def download_and_unzip_gdrive(config, rootpath, hot_run=True, disable_progress=False): """ - download_and_unzip_gdrive(config, root_path, dest_path, hot_run=True, + download_and_unzip_gdrive(config, rootpath, dest_path, hot_run=True, disable_progress=False) Function to download and unzip the data from google drive @@ -178,7 +175,7 @@ def download_and_unzip_gdrive(config, root_path, hot_run=True, disable_progress= ------ config : Dict Configuration data for the category to download - root_path : str + rootpath : str Absolute path of the repository hot_run : Bool (default True) When true the data are downloaded @@ -191,8 +188,8 @@ def download_and_unzip_gdrive(config, root_path, hot_run=True, disable_progress= True when download is successful, False otherwise """ resource = config["category"] - file_path = get_path(root_path, "tempfile.zip") - destination = get_path(BASE_DIR, config["destination"]) + file_path = os.path.join(rootpath, "tempfile.zip") + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["gdrive"] # retrieve file_id from path @@ -219,7 +216,8 @@ def download_and_unzip_gdrive(config, root_path, hot_run=True, disable_progress= # if hot run enabled if hot_run: # remove file - get_path(file_path).unlink(missing_ok=True) + if os.path.exists(file_path): + os.remove(file_path) # download file from google drive gdd.download_file_from_google_drive( file_id=file_id, @@ -240,10 +238,10 @@ def download_and_unzip_gdrive(config, root_path, hot_run=True, disable_progress= def download_and_unzip_protectedplanet( - config, root_path, attempts=3, hot_run=True, disable_progress=False + config, rootpath, attempts=3, hot_run=True, disable_progress=False ): """ - download_and_unzip_protectedplanet(config, root_path, dest_path, + download_and_unzip_protectedplanet(config, rootpath, dest_path, hot_run=True, disable_progress=False) Function to download and unzip the data by category from protectedplanet @@ -252,7 +250,7 @@ def download_and_unzip_protectedplanet( ------ config : Dict Configuration data for the category to download - root_path : str + rootpath : str Absolute path of the repository attempts : int (default 3) Number of attempts to download the data by month. @@ -268,8 +266,8 @@ def download_and_unzip_protectedplanet( True when download is successful, False otherwise """ resource = config["category"] - file_path = get_path(root_path, "tempfile_wpda.zip") - destination = get_path(BASE_DIR, config["destination"]) + file_path = os.path.join(rootpath, "tempfile_wpda.zip") + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["protectedplanet"] def get_first_day_of_month(date): @@ -284,7 +282,8 @@ def get_first_day_of_previous_month(date): ) if hot_run: - get_path(file_path).unlink(missing_ok=True) + if os.path.exists(file_path): + os.remove(file_path) downloaded = False @@ -321,17 +320,17 @@ def get_first_day_of_previous_month(date): for fzip in zip_files: # final path of the file try: - inner_zipname = get_path(destination, fzip) + inner_zipname = os.path.join(destination, fzip) zip_obj.extract(fzip, path=destination) - dest_nested = get_path(destination, fzip.split(".")[0]) + dest_nested = os.path.join(destination, fzip.split(".")[0]) with ZipFile(inner_zipname, "r") as nested_zip: nested_zip.extractall(path=dest_nested) # remove inner zip file - get_path(inner_zipname).unlink(missing_ok=True) + os.remove(inner_zipname) logger.info(f"{resource} - Successfully unzipped file '{fzip}'") except: @@ -341,7 +340,7 @@ def get_first_day_of_previous_month(date): # close and remove outer zip file zip_obj.close() - get_path(file_path).unlink(missing_ok=True) + os.remove(file_path) logger.info( f"Downloaded resource '{resource_iter}' from cloud '{url_iter}'." @@ -349,9 +348,9 @@ def get_first_day_of_previous_month(date): downloaded = True break - except Exception as e: + except: logger.warning( - f"Failed download resource '{resource_iter}' from cloud '{url_iter}' with exception message '{e}'." + f"Failed download resource '{resource_iter}' from cloud '{url_iter}'." ) current_first_day = get_first_day_of_previous_month(current_first_day) @@ -392,7 +391,8 @@ def download_and_unpack( True when download is successful, False otherwise """ if hot_run: - get_path(file_path).unlink(missing_ok=True) + if os.path.exists(file_path): + os.remove(file_path) try: logger.info(f"Downloading resource '{resource}' from cloud '{url}'.") @@ -404,21 +404,19 @@ def download_and_unpack( # then unzip it and remove the original file if unzip: with ZipFile(file_path, "r") as zipfile: - zipfile.extractall(path=destination) + zipfile.extractall(destination) - get_path(file_path).unlink(missing_ok=True) + os.remove(file_path) logger.info(f"Downloaded resource '{resource}' from cloud '{url}'.") return True - except Exception as e: - logger.warning( - f"Failed download resource '{resource}' from cloud '{url}' with exception message '{e}'." - ) + except: + logger.warning(f"Failed download resource '{resource}' from cloud '{url}'.") return False -def download_and_unzip_direct(config, hot_run=True, disable_progress=False): +def download_and_unzip_direct(config, rootpath, hot_run=True, disable_progress=False): """ - download_and_unzip_direct(config, dest_path, hot_run=True, + download_and_unzip_direct(config, rootpath, dest_path, hot_run=True, disable_progress=False) Function to download the data by category from a direct url with no processing. @@ -428,6 +426,8 @@ def download_and_unzip_direct(config, hot_run=True, disable_progress=False): ------ config : Dict Configuration data for the category to download + rootpath : str + Absolute path of the repository hot_run : Bool (default True) When true the data are downloaded When false, the workflow is run without downloading and unzipping @@ -439,10 +439,10 @@ def download_and_unzip_direct(config, hot_run=True, disable_progress=False): True when download is successful, False otherwise """ resource = config["category"] - destination = get_path(BASE_DIR, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["direct"] - file_path = get_path(destination, get_path(url).name) + file_path = os.path.join(destination, os.path.basename(url)) unzip = config.get("unzip", False) @@ -457,10 +457,10 @@ def download_and_unzip_direct(config, hot_run=True, disable_progress=False): def download_and_unzip_hydrobasins( - config, hot_run=True, disable_progress=False + config, rootpath, hot_run=True, disable_progress=False ): """ - download_and_unzip_basins(config, dest_path, hot_run=True, + download_and_unzip_basins(config, rootpath, dest_path, hot_run=True, disable_progress=False) Function to download and unzip the data for hydrobasins from HydroBASINS database @@ -480,6 +480,8 @@ def download_and_unzip_hydrobasins( ------ config : Dict Configuration data for the category to download + rootpath : str + Absolute path of the repository hot_run : Bool (default True) When true the data are downloaded When false, the workflow is run without downloading and unzipping @@ -491,7 +493,7 @@ def download_and_unzip_hydrobasins( True when download is successful, False otherwise """ resource = config["category"] - destination = get_path(BASE_DIR, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url_templ = config["urls"]["hydrobasins"]["base_url"] suffix_list = config["urls"]["hydrobasins"]["suffixes"] @@ -502,7 +504,7 @@ def download_and_unzip_hydrobasins( for rg in suffix_list: url = url_templ + "hybas_" + rg + "_lev" + level_code + "_v1c.zip" - file_path = get_path(destination, get_path(url).name) + file_path = os.path.join(destination, os.path.basename(url)) all_downloaded &= download_and_unpack( url=url, @@ -518,9 +520,9 @@ def download_and_unzip_hydrobasins( return all_downloaded -def download_and_unzip_post(config, hot_run=True, disable_progress=False): +def download_and_unzip_post(config, rootpath, hot_run=True, disable_progress=False): """ - download_and_unzip_post(config, dest_path, hot_run=True, + download_and_unzip_post(config, rootpath, dest_path, hot_run=True, disable_progress=False) Function to download the data by category from a post request. @@ -529,6 +531,8 @@ def download_and_unzip_post(config, hot_run=True, disable_progress=False): ------ config : Dict Configuration data for the category to download + rootpath : str + Absolute path of the repository hot_run : Bool (default True) When true the data are downloaded When false, the workflow is run without downloading and unzipping @@ -540,17 +544,18 @@ def download_and_unzip_post(config, hot_run=True, disable_progress=False): True when download is successful, False otherwise """ resource = config["category"] - destination = get_path(BASE_DIR, config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) # load data for post method postdata = config["urls"]["post"] # remove url feature url = postdata.pop("url") - file_path = get_path(destination, get_path(url).name) + file_path = os.path.join(destination, os.path.basename(url)) if hot_run: - get_path(file_path).unlink(missing_ok=True) + if os.path.exists(file_path): + os.remove(file_path) # try: logger.info(f"Downloading resource '{resource}' from cloud '{url}'.") @@ -566,9 +571,9 @@ def download_and_unzip_post(config, hot_run=True, disable_progress=False): # then unzip it and remove the original file if config.get("unzip", False): with ZipFile(file_path, "r") as zipfile: - zipfile.extractall(path=destination) + zipfile.extractall(destination) - get_path(file_path).unlink(missing_ok=True) + os.remove(file_path) logger.info(f"Downloaded resource '{resource}' from cloud '{url}'.") # except: # logger.warning(f"Failed download resource '{resource}' from cloud '{url}'.") @@ -788,17 +793,17 @@ def datafiles_retrivedatabundle(config): def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): - basins_path = get_path(BASE_DIR, config_hydrobasin["destination"]) - output_fl = get_path(BASE_DIR, config_hydrobasin["output"][0]) + basins_path = os.path.join(BASE_DIR, config_hydrobasin["destination"]) + output_fl = os.path.join(BASE_DIR, config_hydrobasin["output"][0]) files_to_merge = [ "hybas_{0:s}_lev{1:02d}_v1c.shp".format(suffix, hydrobasins_level) for suffix in config_hydrobasin["urls"]["hydrobasins"]["suffixes"] ] gpdf_list = [None] * len(files_to_merge) - logger.info(f"Merging hydrobasins files into: {output_fl}") + logger.info("Merging hydrobasins files into: " + output_fl) for i, f_name in tqdm(enumerate(files_to_merge)): - gpdf_list[i] = gpd.read_file(get_path(basins_path, f_name)) + gpdf_list[i] = gpd.read_file(os.path.join(basins_path, f_name)) fl_merged = gpd.GeoDataFrame(pd.concat(gpdf_list)).drop_duplicates( subset="HYBAS_ID", ignore_index=True ) @@ -807,12 +812,15 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): if __name__ == "__main__": if "snakemake" not in globals(): + + from _helpers import mock_snakemake + snakemake = mock_snakemake("retrieve_databundle_light") # TODO Make logging compatible with progressbar (see PR #102, PyPSA-Eur) configure_logging(snakemake) - root_path = "." + rootpath = "." tutorial = snakemake.params.tutorial countries = snakemake.params.countries logger.info(f"Retrieving data for {len(countries)} countries.") @@ -853,13 +861,11 @@ def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): try: download_and_unzip = globals()[f"download_and_unzip_{host}"] if download_and_unzip( - config_bundles[b_name], root_path, disable_progress=disable_progress + config_bundles[b_name], rootpath, disable_progress=disable_progress ): downloaded_bundle = True - except Exception as e: - logger.warning( - f"Error in downloading bundle {b_name} - host {host} - with exception message '{e}'" - ) + except Exception: + logger.warning(f"Error in downloading bundle {b_name} - host {host}") if downloaded_bundle: downloaded_bundles.append(b_name) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 5dc3c5973..92c3dd340 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -84,6 +84,7 @@ 4. Optionally, if an integer were provided for the wildcard ``{simpl}`` (e.g. ``networks/elec_s500.nc``), the network is clustered to this number of clusters with the routines from the ``cluster_network`` rule with the function ``cluster_network.cluster(...)``. This step is usually skipped! """ +import os import sys from functools import reduce @@ -96,7 +97,6 @@ configure_logging, create_logger, get_aggregation_strategies, - mock_snakemake, update_p_nom_max, ) from add_electricity import load_costs @@ -962,6 +962,8 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("simplify_network", simpl="") configure_logging(snakemake) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index fc86d8988..a9bbfbaa1 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -78,18 +78,14 @@ the rule :mod:`solve_network`. """ import logging +import os import re +from pathlib import Path import numpy as np import pandas as pd import pypsa -from _helpers import ( - build_directory, - configure_logging, - create_logger, - mock_snakemake, - override_component_attrs, -) +from _helpers import configure_logging, create_logger, override_component_attrs from pypsa.descriptors import get_switchable_as_dense as get_as_dense from pypsa.linopf import ( define_constraints, @@ -970,6 +966,8 @@ def solve_network(n, config, solving={}, opts="", **kwargs): if __name__ == "__main__": if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake( "solve_network", simpl="", @@ -982,7 +980,7 @@ def solve_network(n, config, solving={}, opts="", **kwargs): tmpdir = snakemake.params.solving.get("tmpdir") if tmpdir is not None: - build_directory(tmpdir, just_parent_directory=False) + Path(tmpdir).mkdir(parents=True, exist_ok=True) opts = snakemake.wildcards.opts.split("-") solving = snakemake.params.solving diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index fa7a7644d..000000000 --- a/test/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/test/conftest.py b/test/conftest.py deleted file mode 100644 index 985692c0b..000000000 --- a/test/conftest.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import pathlib -import shutil - -import pypsa -import pytest -import yaml - -_content_temp_file = "content" -_name_temp_file = "hello.txt" -_temp_content_dir = "temp_content_dir" -_sub_temp_content_dir = "sub_temp_content_dir" - - -@pytest.fixture(scope="function") -def get_temp_file(tmpdir): - p = pathlib.Path(tmpdir, _name_temp_file) - p.write_text(_content_temp_file) - yield p - pathlib.Path(p).unlink(missing_ok=True) - - -@pytest.fixture(scope="function") -def get_temp_folder(tmpdir): - temp_content_dir = tmpdir.join(_temp_content_dir) - sub_temp_content_dir = temp_content_dir.join(_sub_temp_content_dir) - yield sub_temp_content_dir - shutil.rmtree(str(sub_temp_content_dir)) - - -@pytest.fixture(scope="function") -def get_power_network_scigrid_de(): - return pypsa.examples.scigrid_de(from_master=True) - - -@pytest.fixture(scope="function") -def get_power_network_ac_dc_meshed(): - return pypsa.examples.ac_dc_meshed(from_master=True) - - -@pytest.fixture(scope="function") -def get_config_dict(): - path_config = pathlib.Path(pathlib.Path.cwd(), "config.default.yaml") - with open(path_config, "r") as file: - config_dict = yaml.safe_load(file) - return config_dict diff --git a/test/test_add_extra_components.py b/test/test_add_extra_components.py deleted file mode 100644 index ee7afe9ed..000000000 --- a/test/test_add_extra_components.py +++ /dev/null @@ -1,137 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import pathlib -import sys - -sys.path.append("./scripts") - -from test.conftest import get_config_dict, get_power_network_scigrid_de - -from add_electricity import load_costs -from add_extra_components import ( - attach_hydrogen_pipelines, - attach_storageunits, - attach_stores, -) - -path_cwd = pathlib.Path.cwd() -path_costs = pathlib.Path(path_cwd, "data", "costs.csv") - - -def test_attach_storageunits(get_config_dict, get_power_network_scigrid_de): - """ - Verify what is returned by attach_storageunits() - """ - config_dict = get_config_dict - config_dict["electricity"]["extendable_carriers"]["StorageUnit"] = ["H2"] - test_network_de = get_power_network_scigrid_de - number_years = test_network_de.snapshot_weightings.objective.sum() / 8760.0 - test_costs = load_costs( - path_costs, - config_dict["costs"], - config_dict["electricity"], - number_years, - ) - - reference_component_dict = { - "Bus": 585, - "Carrier": 1, - "Line": 852, - "LineType": 34, - "Transformer": 96, - "TransformerType": 14, - "Load": 489, - "Generator": 1423, - "StorageUnit": 623, - } - attach_storageunits(test_network_de, test_costs, config_dict) - - output_component_dict = {} - for c in test_network_de.iterate_components( - list(test_network_de.components.keys())[2:] - ): - output_component_dict[c.name] = len(c.df) - - assert output_component_dict == reference_component_dict - - -def test_attach_stores(get_config_dict, get_power_network_scigrid_de): - """ - Verify what is returned by attach_stores() - """ - config_dict = get_config_dict - config_dict["renewable"]["csp"]["csp_model"] = "simple" - test_network_de = get_power_network_scigrid_de - number_years = test_network_de.snapshot_weightings.objective.sum() / 8760.0 - test_costs = load_costs( - path_costs, - config_dict["costs"], - config_dict["electricity"], - number_years, - ) - - reference_component_dict = { - "Bus": 1755, - "Carrier": 2, - "Line": 852, - "LineType": 34, - "Transformer": 96, - "TransformerType": 14, - "Link": 2340, - "Load": 489, - "Generator": 1423, - "StorageUnit": 38, - "Store": 1170, - } - test_network_de.buses["country"] = "DE" - attach_stores(test_network_de, test_costs, config_dict) - - output_component_dict = {} - for c in test_network_de.iterate_components( - list(test_network_de.components.keys())[2:] - ): - output_component_dict[c.name] = len(c.df) - - assert output_component_dict == reference_component_dict - - -def test_attach_hydrogen_pipelines(get_config_dict, get_power_network_scigrid_de): - """ - Verify what is returned by attach_hydrogen_pipelines() - """ - config_dict = get_config_dict - config_dict["electricity"]["extendable_carriers"]["Link"] = ["H2 pipeline"] - test_network_de = get_power_network_scigrid_de - number_years = test_network_de.snapshot_weightings.objective.sum() / 8760.0 - test_costs = load_costs( - path_costs, - config_dict["costs"], - config_dict["electricity"], - number_years, - ) - - reference_component_dict = { - "Bus": 585, - "Line": 852, - "LineType": 34, - "Transformer": 96, - "TransformerType": 14, - "Link": 705, - "Load": 489, - "Generator": 1423, - "StorageUnit": 38, - } - attach_hydrogen_pipelines(test_network_de, test_costs, config_dict) - - output_component_dict = {} - for c in test_network_de.iterate_components( - list(test_network_de.components.keys())[2:] - ): - output_component_dict[c.name] = len(c.df) - - assert output_component_dict == reference_component_dict diff --git a/test/test_base_network.py b/test/test_base_network.py deleted file mode 100644 index 0fc035683..000000000 --- a/test/test_base_network.py +++ /dev/null @@ -1,491 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import pathlib -import sys - -import numpy as np -import pandas as pd - -sys.path.append("./scripts") - -from test.conftest import get_config_dict - -from _helpers import get_path -from base_network import ( - _get_linetype_by_voltage, - _get_linetypes_config, - _load_buses_from_osm, - _load_converters_from_osm, - _load_lines_from_osm, - _load_transformers_from_osm, - _set_electrical_parameters_converters, - _set_electrical_parameters_dc_lines, - _set_electrical_parameters_lines, - _set_electrical_parameters_links, - _set_electrical_parameters_transformers, - get_country, -) - -# Common references - -# ---> buses - -df_buses_input = pd.DataFrame( - { - "bus_id": 0, - "station_id": 0, - "voltage": 161000, - "dc": False, - "symbol": "substation", - "under_construction": False, - "tag_substation": "transmission", - "tag_area": 0.0, - "lon": 2.5914, - "lat": 9.3321, - "country": "BJ", - "geometry": "POINT (2.5914 9.3321)", - "substation_lv": True, - }, - index=[0], -) - -df_buses_reference = pd.DataFrame( - { - "bus_id": "0", - "v_nom": 161.0, - "symbol": "substation", - "under_construction": False, - "tag_substation": "transmission", - "tag_area": 0.0, - "lon": 2.5914, - "lat": 9.3321, - "country": "BJ", - "geometry": "POINT (2.5914 9.3321)", - "substation_lv": True, - "carrier": "AC", - "x": 2.5914, - "y": 9.3321, - }, - index=[0], -).set_index("bus_id") - -# ---> converters - -df_converters_input = pd.DataFrame( - { - "index": 0, - "converter_id": "convert_20_41", - "bus0": "41", - "bus1": "42", - "underground": False, - "under_construction": False, - "country": "US", - "geometry": "LINESTRING(-122.3787 37.6821, -122.3777 37.6831)", - }, - index=[0], -) - -df_converters_reference = pd.DataFrame( - { - "converter_id": "convert_20_41", - "index": 0, - "bus0": "41", - "bus1": "42", - "underground": False, - "under_construction": False, - "country": "US", - "geometry": "LINESTRING(-122.3787 37.6821, -122.3777 37.6831)", - "carrier": "B2B", - "dc": True, - }, - index=[0], -).set_index("converter_id") - -# ---> lines - -df_lines_input = pd.DataFrame( - { - "line_id": ["204361221-1_0", "204361287-1_1"], - "tag_frequency": [50.0, 0.0], - "tag_type": ["line", "line"], - "voltage": [161000, 178658], - "bus0": ["111", "111"], - "bus1": ["0", "0"], - "circuits": (3.0, 3.0), - "length": [110071.89434240988, 118723.89434240988], - "underground": [False, False], - "under_construction": [False, False], - "dc": [False, False], - "country": ["BJ", "BJ"], - "geometry": [ - "LINESTRING (2.6594 10.2042, 2.6594451 10.2042341)", - "LINESTRING (2.6594 10.2042, 2.6594451 10.2042341)", - ], - "bounds": [ - "MULTIPOINT ((2.6594 10.2042), (2.5914 9.3321))", - "MULTIPOINT ((2.6594 10.2042), (2.5914 9.3321))", - ], - "bus_0_coors": ["POINT (2.6594 10.2042)", "POINT (2.6594 10.2042)"], - "bus_1_coors": ["POINT (2.5914 9.3321)", "POINT (2.5914 9.3321)"], - "bus0_lon": [2.6594, 2.6594], - "bus0_lat": [10.2042, 10.2042], - "bus1_lon": [2.5914, 2.5914], - "bus1_lat": [9.3321, 9.3321], - } -) - -df_lines_reference = pd.DataFrame( - { - "line_id": ["204361221-1_0", "204361287-1_1"], - "tag_frequency": [50.0, 0.0], - "tag_type": ["line", "line"], - "v_nom": [161.0, 178.658], - "bus0": ["111", "111"], - "bus1": ["0", "0"], - "num_parallel": [3.0, 3.0], - "length": [110.07189434240988, 118.72389434240988], - "underground": [False, False], - "under_construction": [False, False], - "dc": [False, False], - "country": ["BJ", "BJ"], - "geometry": [ - "LINESTRING (2.6594 10.2042, 2.6594451 10.2042341)", - "LINESTRING (2.6594 10.2042, 2.6594451 10.2042341)", - ], - "bounds": [ - "MULTIPOINT ((2.6594 10.2042), (2.5914 9.3321))", - "MULTIPOINT ((2.6594 10.2042), (2.5914 9.3321))", - ], - "bus_0_coors": ["POINT (2.6594 10.2042)", "POINT (2.6594 10.2042)"], - "bus_1_coors": ["POINT (2.5914 9.3321)", "POINT (2.5914 9.3321)"], - "bus0_lon": [2.6594, 2.6594], - "bus0_lat": [10.2042, 10.2042], - "bus1_lon": [2.5914, 2.5914], - "bus1_lat": [9.3321, 9.3321], - } -).set_index("line_id") - -lines_ac_reference = pd.DataFrame( - { - "tag_frequency": 50.0, - "tag_type": "line", - "v_nom": 161.0, - "bus0": "111", - "bus1": "0", - "num_parallel": 3.0, - "length": 110.07189434240988, - "underground": False, - "under_construction": False, - "dc": False, - "country": "BJ", - "geometry": "LINESTRING (2.6594 10.2042, 2.6594451 10.2042341)", - "bounds": "MULTIPOINT ((2.6594 10.2042), (2.5914 9.3321))", - "bus_0_coors": "POINT (2.6594 10.2042)", - "bus_1_coors": "POINT (2.5914 9.3321)", - "bus0_lon": 2.6594, - "bus0_lat": 10.2042, - "bus1_lon": 2.5914, - "bus1_lat": 9.3321, - "carrier": "AC", - "type": "243-AL1/39-ST1A 20.0", - "s_max_pu": 0.7, - }, - index=[0], -).set_index("tag_frequency") - -lines_dc_reference = pd.DataFrame( - { - "tag_frequency": 0.0, - "tag_type": "line", - "v_nom": 178.658, - "bus0": "111", - "bus1": "0", - "num_parallel": 3.0, - "length": 118.72389434240988, - "underground": False, - "under_construction": False, - "dc": True, - "country": "BJ", - "geometry": "LINESTRING (2.6594 10.2042, 2.6594451 10.2042341)", - "bounds": "MULTIPOINT ((2.6594 10.2042), (2.5914 9.3321))", - "bus_0_coors": "POINT (2.6594 10.2042)", - "bus_1_coors": "POINT (2.5914 9.3321)", - "bus0_lon": 2.6594, - "bus0_lat": 10.2042, - "bus1_lon": 2.5914, - "bus1_lat": 9.3321, - "carrier": "DC", - "type": "HVDC XLPE 1000", - "s_max_pu": 0.7, - }, - index=[0], -).set_index("tag_frequency") - - -df_transformers_input = pd.DataFrame( - { - "line_id": "transf_1_0", - "bus0": "1", - "bus1": "2", - "voltage_bus0": 161000, - "voltage_bus1": 330000, - "country": "BJ", - "geometry": "LINESTRING(2.648 6.7394, 2.649 6.7404)", - "bounds": "MULTIPOINT((2.648 6.7394), (2.649 6.7404))", - "bus_0_coors": "POINT(2.648 6.7394)", - "bus_1_coors": "POINT(2.649 6.7404)", - "bus0_lon": 2.648, - "bus0_lat": 6.7394, - "bus1_lon": 2.649, - "bus1_lat": 6.7404, - }, - index=[0], -) - -df_transformers_reference = pd.DataFrame( - { - "transformer_id": "transf_1_0", - "bus0": "1", - "bus1": "2", - "voltage_bus0": 161000, - "voltage_bus1": 330000, - "country": "BJ", - "geometry": "LINESTRING(2.648 6.7394, 2.649 6.7404)", - "bounds": "MULTIPOINT((2.648 6.7394), (2.649 6.7404))", - "bus_0_coors": "POINT(2.648 6.7394)", - "bus_1_coors": "POINT(2.649 6.7404)", - "bus0_lon": 2.648, - "bus0_lat": 6.7394, - "bus1_lon": 2.649, - "bus1_lat": 6.7404, - }, - index=[0], -).set_index("transformer_id") - - -def test_get_country(): - """ - Verify what returned by get_country() - """ - data_list = [['"country"=>"NG"'], ['"country"=>"CH"'], ['"country"=>"AU"']] - df_exercise_with_tags = pd.DataFrame(data_list, columns=["tags"]) - df_exercise_no_tags = pd.DataFrame(data_list, columns=["other"]) - series_with_tags = get_country(df_exercise_with_tags) - reference_series_with_tags = pd.Series(["NG", "CH", "AU"]) - comparison_series_with_tags = series_with_tags.compare(reference_series_with_tags) - series_no_tags = get_country(df_exercise_no_tags) - reference_series_no_tags = pd.Series([np.nan, np.nan, np.nan]) - comparison_series_no_tags = series_no_tags.compare(reference_series_no_tags) - assert comparison_series_with_tags.size == 0 - assert comparison_series_no_tags.size == 0 - - -def test_load_buses_from_osm(tmpdir): - """ - Verify what returned by _load_buses_from_osm. - """ - file_path = get_path(tmpdir, "buses_exercise.csv") - df_buses_input.to_csv(file_path) - df_buses_output = _load_buses_from_osm(file_path) - df_buses_comparison = df_buses_output.compare(df_buses_reference) - pathlib.Path.unlink(file_path) - assert df_buses_comparison.empty - - -def test_load_lines_from_osm(tmpdir): - """ - Verify what returned by _load_lines_from_osm. - """ - file_path = get_path(tmpdir, "lines_exercise.csv") - df_lines_input.to_csv(file_path) - df_lines_output = _load_lines_from_osm(file_path) - df_lines_comparison = df_lines_output.compare(df_lines_reference) - pathlib.Path.unlink(file_path) - assert df_lines_comparison.empty - - -def test_load_transformers_from_osm(tmpdir): - """ - Verify what returned by _load_transformers_from_osm. - """ - file_path = get_path(tmpdir, "transformers_exercise.csv") - df_transformers_input.to_csv(file_path, index=False) - df_transformers_output = _load_transformers_from_osm(file_path) - df_transformers_comparison = df_transformers_output.compare( - df_transformers_reference - ) - pathlib.Path.unlink(file_path) - assert df_transformers_comparison.empty - - -def test_load_converters_from_osm(tmpdir): - """ - Verify what returned by _load_converters_from_osm. - """ - file_path = get_path(tmpdir, "converters_exercise.csv") - df_converters_input.to_csv(file_path, index=False) - df_converters_output = _load_converters_from_osm(file_path) - df_converters_comparison = df_converters_output.compare(df_converters_reference) - pathlib.Path.unlink(file_path) - assert df_converters_comparison.empty - - -def test_get_linetypes_config(get_config_dict): - """ - Verify what returned by _get_linetypes_config. - """ - config_dict = get_config_dict - output_dict_ac = _get_linetypes_config( - config_dict["lines"]["ac_types"], config_dict["electricity"]["voltages"] - ) - output_dict_dc = _get_linetypes_config( - config_dict["lines"]["dc_types"], config_dict["electricity"]["voltages"] - ) - assert output_dict_ac == config_dict["lines"]["ac_types"] - assert output_dict_dc == config_dict["lines"]["dc_types"] - - -def test_get_linetype_by_voltage(get_config_dict): - """ - Verify what returned by _get_linetype_by_voltage. - """ - config_dict = get_config_dict - v_nom_list = [ - 50.0, - 101.0, - 180.0, - 210.0, - 220.0, - 225.0, - 285.0, - 300.0, - 333.0, - 390.0, - 600.0, - 750.0, - 800.0, - ] - - line_type_list = [] - - for v_nom in v_nom_list: - line_type_list.append( - _get_linetype_by_voltage(v_nom, config_dict["lines"]["ac_types"]) - ) - - assert line_type_list == [ - "243-AL1/39-ST1A 20.0", - "243-AL1/39-ST1A 20.0", - "Al/St 240/40 2-bundle 220.0", - "Al/St 240/40 2-bundle 220.0", - "Al/St 240/40 2-bundle 220.0", - "Al/St 240/40 2-bundle 220.0", - "Al/St 240/40 3-bundle 300.0", - "Al/St 240/40 3-bundle 300.0", - "Al/St 240/40 3-bundle 300.0", - "Al/St 240/40 4-bundle 380.0", - "Al/St 240/40 4-bundle 380.0", - "Al/St 560/50 4-bundle 750.0", - "Al/St 560/50 4-bundle 750.0", - ] - - -def test_set_electrical_parameters_lines(get_config_dict, tmpdir): - """ - Verify what returned by _set_electrical_parameters_lines. - """ - config_dict = get_config_dict - file_path = get_path(tmpdir, "lines_exercise.csv") - df_lines_input.to_csv(file_path) - df_lines_output = _load_lines_from_osm(file_path).reset_index(drop=True) - df_lines_output_ac = df_lines_output[ - df_lines_output.tag_frequency.astype(float) != 0 - ].copy() - df_lines_output_dc = df_lines_output[ - df_lines_output.tag_frequency.astype(float) == 0 - ].copy() - lines_ac = _set_electrical_parameters_lines( - config_dict["lines"], config_dict["electricity"]["voltages"], df_lines_output_ac - ).set_index("tag_frequency") - lines_dc = _set_electrical_parameters_dc_lines( - config_dict["lines"], config_dict["electricity"]["voltages"], df_lines_output_dc - ).set_index("tag_frequency") - df_lines_ac_comparison = lines_ac.compare(lines_ac_reference) - df_lines_dc_comparison = lines_dc.compare(lines_dc_reference) - pathlib.Path.unlink(file_path) - assert df_lines_ac_comparison.empty - assert df_lines_dc_comparison.empty - - -def test_set_electrical_parameters_links(get_config_dict, tmpdir): - """ - Verify what returned by _set_electrical_parameters_links. - """ - config_dict = get_config_dict - file_path = get_path(tmpdir, "lines_exercise.csv") - df_lines_input.to_csv(file_path) - df_lines_output = _load_lines_from_osm(file_path).reset_index(drop=True) - df_lines_output_dc = df_lines_output[ - df_lines_output.tag_frequency.astype(float) == 0 - ].copy() - lines_dc = _set_electrical_parameters_dc_lines( - config_dict["lines"], config_dict["electricity"]["voltages"], df_lines_output_dc - ) - new_lines_dc = _set_electrical_parameters_links( - config_dict["links"], lines_dc - ).set_index("tag_frequency") - new_lines_dc_reference = lines_dc_reference.copy(deep=True) - new_lines_dc_reference["p_max_pu"] = config_dict["links"]["p_max_pu"] - new_lines_dc_reference["p_min_pu"] = -config_dict["links"]["p_max_pu"] - pathlib.Path.unlink(file_path) - df_comparison = new_lines_dc.compare(new_lines_dc_reference) - assert df_comparison.empty - - -def test_set_electrical_parameters_transformers(get_config_dict, tmpdir): - """ - Verify what returned by _set_electrical_parameters_transformers. - """ - config_dict = get_config_dict - file_path = get_path(tmpdir, "transformers_exercise.csv") - df_transformers_input.to_csv(file_path, index=False) - df_transformers_output = _load_transformers_from_osm(file_path) - df_transformers_parameters = _set_electrical_parameters_transformers( - config_dict["transformers"], df_transformers_output - ) - df_transformers_parameters_reference = df_transformers_reference.copy(deep=True) - df_transformers_parameters_reference["x"] = config_dict["transformers"]["x"] - df_transformers_parameters_reference["s_nom"] = config_dict["transformers"]["s_nom"] - df_transformers_parameters_reference["type"] = config_dict["transformers"]["type"] - pathlib.Path.unlink(file_path) - df_comparison = df_transformers_parameters.compare( - df_transformers_parameters_reference - ) - assert df_comparison.empty - - -def test_set_electrical_parameters_converters(get_config_dict, tmpdir): - """ - Verify what returned by _set_electrical_parameters_converters. - """ - config_dict = get_config_dict - file_path = get_path(tmpdir, "converters_exercise.csv") - df_converters_input.to_csv(file_path, index=False) - df_converters_output = _load_converters_from_osm(file_path) - df_converters_parameters = _set_electrical_parameters_converters( - config_dict["links"], df_converters_output - ) - df_converters_parameters_reference = df_converters_reference.copy(deep=True) - df_converters_parameters_reference["p_max_pu"] = config_dict["links"]["p_max_pu"] - df_converters_parameters_reference["p_min_pu"] = -config_dict["links"]["p_max_pu"] - df_converters_parameters_reference["p_nom"] = 2000 - df_converters_parameters_reference["under_construction"] = False - df_converters_parameters_reference["underground"] = False - pathlib.Path.unlink(file_path) - df_comparison = df_converters_parameters.compare(df_converters_parameters_reference) - assert df_comparison.empty diff --git a/test/test_build_demand_profiles.py b/test/test_build_demand_profiles.py deleted file mode 100644 index d37ee0d59..000000000 --- a/test/test_build_demand_profiles.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import pathlib -import sys - -sys.path.append("./scripts") - -from test.conftest import get_config_dict - -from _helpers import create_country_list, get_path -from build_demand_profiles import get_gegis_regions, get_load_paths_gegis - -path_cwd = pathlib.Path.cwd() - - -def test_get_gegis_regions(): - """ - Verify what returned by get_gegis_regions. - """ - output_regions = get_gegis_regions(["NG", "IT"]) - assert output_regions == ["Africa", "Europe"] - - -def test_get_load_paths_gegis(get_config_dict): - """ - Verify what returned by get_load_paths_gegis. - """ - config_dict = get_config_dict - load_data_paths = get_load_paths_gegis("data", config_dict) - reference_list = [ - get_path("data", "ssp2-2.6", "2030", "era5_2013", "Africa.nc"), - ] - print(load_data_paths) - assert load_data_paths == reference_list diff --git a/test/test_build_powerplants.py b/test/test_build_powerplants.py deleted file mode 100644 index 34a118c9a..000000000 --- a/test/test_build_powerplants.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import pathlib -import sys - -import pandas as pd -import pytest -import yaml - -sys.path.append("./scripts") - -from test.conftest import get_config_dict - -from build_powerplants import add_power_plants, replace_natural_gas_technology - -path_cwd = pathlib.Path.cwd() - - -def test_replace_natural_gas_technology(): - """ - Verify what returned by replace_natural_gas_technology. - """ - input_df = pd.DataFrame( - { - "Fueltype": [ - "Natural Gas", - "Oil", - "Natural Gas", - "Natural Gas", - "Natural Gas", - "Natural Gas", - "Natural Gas", - "Natural Gas", - "Natural Gas", - "Natural Gas", - "Natural Gas", - "Natural Gas", - "Hydro", - ], - "Technology": [ - "Steam Turbine", - "Combustion Engine", - "NG", - "Ng", - "NG/FO", - "Ng/Fo", - "NG/D", - "LNG", - "CCGT/D", - "CCGT/FO", - "LCCGT", - "CCGT/Fo", - "Reservoir", - ], - } - ) - - reference_df = pd.DataFrame( - { - "Fueltype": [ - "CCGT", - "Oil", - "CCGT", - "CCGT", - "OCGT", - "OCGT", - "OCGT", - "OCGT", - "CCGT", - "CCGT", - "CCGT", - "CCGT", - "Hydro", - ], - "Technology": [ - "CCGT", - "Combustion Engine", - "CCGT", - "CCGT", - "OCGT", - "OCGT", - "OCGT", - "OCGT", - "CCGT", - "CCGT", - "CCGT", - "CCGT", - "Reservoir", - ], - } - ) - modified_df = replace_natural_gas_technology(input_df) - comparison_df = modified_df.compare(reference_df) - assert comparison_df.empty - - -@pytest.mark.parametrize( - "strategy,expected", - [("replace", (4, 19)), ("false", (34, 18)), ("merge", (38, 20))], -) -def test_add_power_plants(get_config_dict, strategy, expected): - """ - Verify what returned by add_power_plants. - """ - config_dict = get_config_dict - custom_powerplants_file_path = pathlib.Path( - path_cwd, "test", "test_data", "custom_NG_powerplants.csv" - ) - pm_config_path = pathlib.Path(path_cwd, "configs", "powerplantmatching_config.yaml") - with open(pm_config_path, "r") as f: - power_plants_config = yaml.safe_load(f) - ppl_query = config_dict["electricity"]["powerplants_filter"] - - config_dict["countries"] = ["NG"] - - powerplants_assignment_strategy = strategy - if isinstance(ppl_query, str): - power_plants_config["main_query"] = ppl_query - countries_names = ["Nigeria"] - power_plants_config["target_countries"] = countries_names - ppl = add_power_plants( - custom_powerplants_file_path, - power_plants_config, - powerplants_assignment_strategy, - countries_names, - ) - # The number of powerplants returned by powerplantmatching - # may vary depending on the version of powerplantmatching - # The numbers below refer to version 0.6.0 - assert ppl.shape == expected diff --git a/test/test_build_shapes.py b/test/test_build_shapes.py deleted file mode 100644 index 9e53b004f..000000000 --- a/test/test_build_shapes.py +++ /dev/null @@ -1,312 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import pathlib -import sys - -import geopandas as gpd -import numpy as np - -sys.path.append("./scripts") - -from _helpers import get_gadm_layer -from build_shapes import ( - _simplify_polys, - add_population_data, - country_cover, - download_WorldPop_API, - download_WorldPop_standard, - get_countries_shapes, - get_gadm_shapes, - load_gdp, - save_to_geojson, -) - -path_cwd = str(pathlib.Path.cwd()) - - -def test_simplify_polys(get_config_dict): - """ - Verify what is returned by _simplify_polys. - """ - - config_dict = get_config_dict - - countries_list = ["NG"] - geo_crs = config_dict["crs"]["geo_crs"] - - update = config_dict["build_shape_options"]["update_file"] - out_logging = config_dict["build_shape_options"]["out_logging"] - contended_flag = config_dict["build_shape_options"]["contended_flag"] - file_prefix = config_dict["build_shape_options"]["gadm_file_prefix"] - gadm_url_prefix = config_dict["build_shape_options"]["gadm_url_prefix"] - gadm_input_file_args = ["data", "gadm"] - - country_shapes_df = get_countries_shapes( - countries_list, - geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, - contended_flag, - update, - out_logging, - ) - - simplified_poly = _simplify_polys(country_shapes_df) - - simplified_poly_df = gpd.GeoDataFrame( - geometry=[ - country_cover( - simplified_poly, eez_shapes=None, out_logging=False, distance=0.02 - ) - ] - ) - simplified_poly_df["area"] = simplified_poly_df.area - simplified_poly_df["centroid"] = simplified_poly_df.centroid - simplified_poly_df["perimeter"] = simplified_poly_df.length - print(simplified_poly_df["perimeter"][0]) - assert np.round(simplified_poly_df.area[0], 6) == 75.750018 - assert ( - str(simplified_poly_df.centroid[0]) - == "POINT (8.100522482086877 9.591585359563023)" - ) - assert np.round(simplified_poly_df["perimeter"][0], 6) == 47.060882 - - -def test_get_countries_shapes(get_config_dict): - """ - Verify what is returned by get_countries_shapes. - """ - - config_dict = get_config_dict - - countries_list = ["XK"] - geo_crs = config_dict["crs"]["geo_crs"] - - update = config_dict["build_shape_options"]["update_file"] - out_logging = config_dict["build_shape_options"]["out_logging"] - contended_flag = config_dict["build_shape_options"]["contended_flag"] - file_prefix = config_dict["build_shape_options"]["gadm_file_prefix"] - gadm_url_prefix = config_dict["build_shape_options"]["gadm_url_prefix"] - gadm_input_file_args = ["data", "gadm"] - - country_shapes_df = get_countries_shapes( - countries_list, - geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, - contended_flag, - update, - out_logging, - ) - - assert country_shapes_df.shape == (1,) - assert country_shapes_df.index.unique().tolist() == ["XK"] - - -def test_country_cover(get_config_dict): - """ - Verify what is returned by country_cover. - """ - - config_dict = get_config_dict - - countries_list = ["NG"] - geo_crs = config_dict["crs"]["geo_crs"] - - update = config_dict["build_shape_options"]["update_file"] - out_logging = config_dict["build_shape_options"]["out_logging"] - contended_flag = config_dict["build_shape_options"]["contended_flag"] - file_prefix = config_dict["build_shape_options"]["gadm_file_prefix"] - gadm_url_prefix = config_dict["build_shape_options"]["gadm_url_prefix"] - gadm_input_file_args = ["data", "gadm"] - - country_shapes_df = get_countries_shapes( - countries_list, - geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, - contended_flag, - update, - out_logging, - ) - - africa_shapes_df = gpd.GeoDataFrame( - geometry=[ - country_cover( - country_shapes_df, eez_shapes=None, out_logging=False, distance=0.02 - ) - ] - ) - africa_shapes_df["area"] = africa_shapes_df.area - africa_shapes_df["centroid"] = africa_shapes_df.centroid - africa_shapes_df["perimeter"] = africa_shapes_df.length - print(africa_shapes_df["perimeter"]) - assert np.round(africa_shapes_df.area[0], 6) == 75.750104 - assert ( - str(africa_shapes_df.centroid[0]) - == "POINT (8.100519548407405 9.59158035236806)" - ) - assert np.round(africa_shapes_df["perimeter"][0], 6) == 47.080743 - - -def test_download_world_pop_standard(get_config_dict): - """ - Verify what is returned by download_WorldPop_standard. - """ - - config_dict = get_config_dict - update_val = config_dict["build_shape_options"]["update_file"] - out_logging_val = config_dict["build_shape_options"]["out_logging"] - - world_pop_input_file, world_pop_file_name = download_WorldPop_standard( - "NG", - year=2020, - update=update_val, - out_logging=out_logging_val, - size_min=300, - ) - assert world_pop_file_name == "nga_ppp_2020_UNadj_constrained.tif" - - -def test_download_world_pop_api(): - """ - Verify what is returned by download_WorldPop_API. - """ - world_pop_input_file, world_pop_file_name = download_WorldPop_API( - "NG", year=2020, size_min=300 - ) - assert world_pop_file_name == "nga_ppp_2020_UNadj_constrained.tif" - - -def test_get_gadm_shapes(get_config_dict): - """ - Verify what is returned by get_gadm_shapes. - """ - config_dict = get_config_dict - - mem_mb = 3096 - - countries_list = ["XK"] - geo_crs = config_dict["crs"]["geo_crs"] - - layer_id = config_dict["build_shape_options"]["gadm_layer_id"] - update = config_dict["build_shape_options"]["update_file"] - out_logging = config_dict["build_shape_options"]["out_logging"] - year = config_dict["build_shape_options"]["year"] - nprocesses = config_dict["build_shape_options"]["nprocesses"] - contended_flag = config_dict["build_shape_options"]["contended_flag"] - worldpop_method = config_dict["build_shape_options"]["worldpop_method"] - gdp_method = config_dict["build_shape_options"]["gdp_method"] - file_prefix = config_dict["build_shape_options"]["gadm_file_prefix"] - gadm_url_prefix = config_dict["build_shape_options"]["gadm_url_prefix"] - gadm_input_file_args = ["data", "gadm"] - - gadm_shapes_df = get_gadm_shapes( - worldpop_method, - gdp_method, - countries_list, - geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, - contended_flag, - mem_mb, - layer_id, - update, - out_logging, - year, - nprocesses=nprocesses, - ) - - assert gadm_shapes_df.shape == (7, 4) - assert gadm_shapes_df.index.unique().tolist() == [f"XK.{x}_1" for x in range(1, 8)] - assert gadm_shapes_df.loc["XK.1_1"]["pop"] == 207473.70381259918 - - -def test_add_population_data(get_config_dict): - """ - Verify what is returned by add_population_data. - """ - config_dict = get_config_dict - - mem_mb = 3096 - - countries_list = ["XK"] - geo_crs = config_dict["crs"]["geo_crs"] - - layer_id = config_dict["build_shape_options"]["gadm_layer_id"] - update = config_dict["build_shape_options"]["update_file"] - out_logging = config_dict["build_shape_options"]["out_logging"] - year = config_dict["build_shape_options"]["year"] - nprocesses = config_dict["build_shape_options"]["nprocesses"] - contended_flag = config_dict["build_shape_options"]["contended_flag"] - worldpop_method = config_dict["build_shape_options"]["worldpop_method"] - file_prefix = config_dict["build_shape_options"]["gadm_file_prefix"] - gadm_url_prefix = config_dict["build_shape_options"]["gadm_url_prefix"] - gadm_input_file_args = ["data", "gadm"] - - mem_read_limit_per_process = mem_mb / nprocesses - - df_gadm = get_gadm_layer( - countries_list, - layer_id, - geo_crs, - file_prefix, - gadm_url_prefix, - gadm_input_file_args, - contended_flag, - update, - out_logging, - ) - - # select and rename columns - df_gadm.rename(columns={"GID_0": "country"}, inplace=True) - - # drop useless columns - df_gadm.drop( - df_gadm.columns.difference(["country", "GADM_ID", "geometry"]), - axis=1, - inplace=True, - errors="ignore", - ) - - add_population_data( - df_gadm, - countries_list, - worldpop_method, - year, - update, - out_logging, - mem_read_limit_per_process, - nprocesses=nprocesses, - ) - - assert np.round(df_gadm["pop"].values[0], 0) == 207474.0 - assert np.round(df_gadm["pop"].values[1], 0) == 208332.0 - assert np.round(df_gadm["pop"].values[2], 0) == 257191.0 - assert np.round(df_gadm["pop"].values[3], 0) == 215703.0 - assert np.round(df_gadm["pop"].values[4], 0) == 610695.0 - assert np.round(df_gadm["pop"].values[5], 0) == 420344.0 - assert np.round(df_gadm["pop"].values[6], 0) == 215316.0 - - -def test_load_gdp(get_config_dict): - """ - Verify what is returned by load_gdp. - """ - config_dict = get_config_dict - - update = config_dict["build_shape_options"]["update_file"] - out_logging = config_dict["build_shape_options"]["out_logging"] - year = config_dict["build_shape_options"]["year"] - name_file_nc = "GDP_PPP_1990_2015_5arcmin_v2.nc" - GDP_tif, name_tif = load_gdp(year, update, out_logging, name_file_nc) - assert name_tif == "GDP_PPP_1990_2015_5arcmin_v2.tif" diff --git a/test/test_data/custom_NG_powerplants.csv b/test/test_data/custom_NG_powerplants.csv deleted file mode 100644 index cacc22d5a..000000000 --- a/test/test_data/custom_NG_powerplants.csv +++ /dev/null @@ -1,5 +0,0 @@ -Name,Fueltype,Technology,Set,Country,Capacity,Efficiency,Duration,Volume_Mm3,DamHeight_m,StorageCapacity_MWh,DateIn,DateRetrofit,DateMothball,DateOut,lat,lon,EIC,projectID,bus -Jebba,hydro,Reservoir,PP,NG,578,,,0,0,0,0,1985,1985,2085,9.1409,4.7896,{nan},"{'GHPT': {'G602885'}, 'GEO': {'GEO-42544'}, 'GPD': {'WRI1000037'}}", -Olorunsogo,CCGT,CCGT,PP,NG,754,,,0,0,0,0,2008,2008,2048,6.885316,3.316268,"{nan, nan}","{'GGPT': {'L406801'}, 'GEO': {'GEO-42564'}, 'GPD': {'WRI1000031'}}", -Gbarain Gas,CCGT,CCGT,PP,NG,252,,,0,0,0,0,2016,2016,2056,5.031067,6.301568,"{nan, nan}","{'GGPT': {'L406786'}, 'GEO': {'GEO-42563'}, 'GPD': {'WRI1000027'}}", -Sapele,CCGT,CCGT,PP,NG,1472,,,0,0,0,0,1978,1978,2018,5.926139,5.644983,"{nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan}","{'GGPT': {'L406809', 'L406808'}, 'GEO': {'GEO-42560'}, 'GPD': {'WRI1000028'}}", diff --git a/test/test_helpers.py b/test/test_helpers.py deleted file mode 100644 index a8c0f5c4f..000000000 --- a/test/test_helpers.py +++ /dev/null @@ -1,616 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import pathlib -import shutil -import sys -from test.conftest import ( - _content_temp_file, - _name_temp_file, - _sub_temp_content_dir, - _temp_content_dir, - get_temp_file, -) - -import fiona -import numpy as np -import pandas as pd - -sys.path.append("./scripts") - -from _helpers import ( - aggregate_fuels, - build_directory, - change_to_script_dir, - country_name_2_two_digits, - download_gadm, - get_base_dir, - get_config_default_path, - get_conv_factors, - get_current_directory_path, - get_gadm_filename, - get_gadm_url, - get_path, - get_path_size, - get_relative_path, - modify_commodity, - normed, - safe_divide, - three_2_two_digits_country, - two_2_three_digits_country, - two_digits_2_name_country, -) - -path_cwd = str(pathlib.Path.cwd()) - - -original_commodity_data = [ - "Biogases", - "Fuelwood", - "of which: fishing", - "Natural gas liquids", - "Naphtha", - "Motor Gasoline", - "Motor gasoline", - "Gasoline-type jet fuel", - "Peat products", - "Peat Products", - "Direct use of geothermal heat", - "Additives and Oxygenates", - "Electricity", - "Animal waste", - "animal waste", - "Refinery gas", - "Refinery Gas", - "Fuel oil", - "Oil shale", - "Oil Shale", - "Lignite", - "Falling water", - "Petroleum coke", - "Petroleum Coke", - "Aviation gasoline", - "Ethane", - "Natural gas (including LNG)", - "Natural gas", - "Natural Gas (including LNG)", - "Other bituminous coal", - "Paraffin waxes", - "Hard coal", - "Coal", - "Hrad coal", - "Coke Oven Gas", - "Gasworks Gas", - "Brown coal briquettes", - "Brown Coal Briquettes", - "Liquefied petroleum gas (LPG)", - "Liquified Petroleum Gas (LPG)", - "Sub-bituminous coal", - "Kerosene-type Jet Fuel", - "Charcoal", - "Heat", - "Gas coke", - "Gas Coke", - "Patent fuel", - "Peat (for fuel use)", - "Peat", - "Coal Tar", - "Biogasoline", - "Coking coal", - "Electricity generating capacity", - "Anthracite", - "Coke oven coke", - "Coke-oven coke", - "Coke Oven Coke", - "Conventional crude oil", - "Crude petroleum", - "Brown coal", - "Lignite brown coal", - "Lignite brown coal- recoverable resources", - "Biodiesel", - "Lubricants", - "Black Liquor", - "Gas Oil/ Diesel Oil", - "Gas Oil/ Diesel Oil ", - "Gas Oil/Diesel Oil", - "Bagasse", - "Direct use of solar thermal heat", - "Bio jet kerosene", - "Blast Furnace Gas", - "Blast furnace gas", - "Bitumen", -] - -modified_commodity_data = [ - "biogases", - "fuelwood", - "of which: fishing", - "natural gas liquids", - "naphtha", - "motor gasoline", - "gasoline-type jet fuel", - "peat products", - "direct use of geothermal heat", - "additives and oxygenates", - "electricity", - "animal waste", - "refinery gas", - "fuel oil", - "oil shale", - "lignite", - "falling water", - "petroleum coke", - "aviation gasoline", - "ethane", - "natural gas (including lng)", - "natural gas", - "other bituminous coal", - "paraffin waxes", - "hard coal", - "coal", - "coke-oven gas", - "gasworks gas", - "brown coal briquettes", - "liquefied petroleum gas (lpg)", - "sub-bituminous coal", - "kerosene-type jet fuel", - "charcoal", - "heat", - "gas coke", - "patent fuel", - "peat (for fuel use)", - "peat", - "coal tar", - "biogasoline", - "coking coal", - "electricity generating capacity", - "anthracite", - "coke-oven coke", - "conventional crude oil", - "crude petroleum", - "brown coal", - "lignite brown coal", - "lignite brown coal - recoverable resources", - "biodiesel", - "lubricants", - "black liquor", - "gas oil/ diesel oil", - "bagasse", - "direct use of solar thermal heat", - "bio jet kerosene", - "blast furnace gas", - "bitumen", -] - -original_commodity_dataframe = pd.DataFrame( - original_commodity_data, columns=["Commodity"] -) -modified_commodity_dataframe = pd.DataFrame( - modified_commodity_data, columns=["Commodity"] -) - - -def test_build_directory(get_temp_folder, tmpdir): - """ - Verify the directory tree returned by build_directory() - - Please note: - -) build_directory(path, just_parent_directory=True) is equivalent to os.makedirs(os.path.dirname(path)). - Given a path tmpdir/temp_content_dir/sub_temp_content_dir, it will create just tmpdir/temp_content_dir/ - -) build_directory(path, just_parent_directory=False) is equivalent to os.makedirs(path). Given a path - tmpdir/temp_content_dir/sub_temp_content_dir, it will create tmpdir/temp_content_dir/sub_temp_content_dir - """ - - # test with pathlib - build_directory(get_temp_folder, just_parent_directory=True) - just_parent_list_pathlib = [] - for root, dirs, files in os.walk(tmpdir): - just_parent_list_pathlib.append(str(get_path(root))) - - assert len(just_parent_list_pathlib) == 2 - assert just_parent_list_pathlib[0] == str(tmpdir) - assert just_parent_list_pathlib[1] == str(tmpdir.join(_temp_content_dir)) - - # remove the temporary folder tmpdir/temp_content_dir/ - shutil.rmtree(pathlib.Path(tmpdir, _temp_content_dir)) - - # test with os.makedirs. Please note for exist_ok=False, - # a FileExistsError is raised if the target directory - # already exists. Hence, setting exist_ok=False ensures - # that the removal with shutil.rmtree was successful - os.makedirs(os.path.dirname(get_temp_folder), exist_ok=False) - just_parent_list_os = [] - for root, dirs, files in os.walk(tmpdir): - just_parent_list_os.append(str(get_path(root))) - - assert just_parent_list_pathlib == just_parent_list_os - - # test with pathlib - build_directory(get_temp_folder, just_parent_directory=False) - full_tree_list_pathlib = [] - for root, dirs, files in os.walk(tmpdir): - full_tree_list_pathlib.append(str(get_path(root))) - - assert len(full_tree_list_pathlib) == 3 - assert full_tree_list_pathlib[0] == str(tmpdir) - assert full_tree_list_pathlib[1] == str(tmpdir.join(_temp_content_dir)) - assert full_tree_list_pathlib[2] == str( - tmpdir.join(_temp_content_dir, _sub_temp_content_dir) - ) - - # remove the temporary folder tmpdir/temp_content_dir/* - shutil.rmtree(pathlib.Path(tmpdir, _temp_content_dir)) - - # test with os.makedirs. Please note for exist_ok=False, - # a FileExistsError is raised if the target directory - # already exists. Hence, setting exist_ok=False ensures - # that the removal with shutil.rmtree was successful - os.makedirs(get_temp_folder, exist_ok=False) - full_tree_list_os = [] - for root, dirs, files in os.walk(tmpdir): - full_tree_list_os.append(str(get_path(root))) - - assert full_tree_list_os == full_tree_list_pathlib - - -def test_change_to_script_dir(): - """ - Verify the path returned by change_to_script_dir() - """ - change_to_script_dir(__file__) - assert str(pathlib.Path.cwd()) == path_cwd + os.sep + "test" - change_to_script_dir(".") - assert str(pathlib.Path.cwd()) == path_cwd - - -def test_get_path(): - """ - Verify the path returned by get_path() - """ - file_name_path_one = get_path( - path_cwd, - "sub_path_1", - "sub_path_2", - "sub_path_3", - "sub_path_4", - "sub_path_5", - "file.nc", - ) - file_name_path_one_list_unpacked = get_path( - path_cwd, - *[ - "sub_path_1", - "sub_path_2", - "sub_path_3", - "sub_path_4", - "sub_path_5", - "file.nc", - ], - ) - path_name_path_two = get_path( - pathlib.Path(__file__).parent, "..", "logs", "rule.log" - ) - assert str(file_name_path_one) == os.path.join( - path_cwd, - "sub_path_1", - "sub_path_2", - "sub_path_3", - "sub_path_4", - "sub_path_5", - "file.nc", - ) - assert str(file_name_path_one_list_unpacked) == os.path.join( - path_cwd, - "sub_path_1", - "sub_path_2", - "sub_path_3", - "sub_path_4", - "sub_path_5", - "file.nc", - ) - assert str(path_name_path_two) == str( - pathlib.Path(__file__).parent.joinpath("..", "logs", "rule.log") - ) - - -def test_get_path_size(get_temp_file): - """ - Verify the path size (in bytes) returned by get_path_size() - """ - path = get_temp_file - file_size = get_path_size(path) - assert file_size == os.stat(path).st_size - assert file_size == len(_content_temp_file) - - -def test_get_base_dir(get_temp_file): - """ - Verify the base directory path returned by get_base_dir() - """ - assert get_base_dir(get_temp_file) == os.path.abspath(os.path.join(os.path.dirname(get_temp_file), os.pardir)) - assert get_base_dir(__file__) == os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - - -def test_get_config_default_path(get_temp_file): - """ - Verify the base directory path returned by get_config_default_path() - """ - base_dir = get_config_default_path(get_temp_file) - assert get_config_default_path(base_dir) == str(get_path(base_dir, "config.default.yaml")) - base_dir = get_config_default_path(__file__) - assert get_config_default_path(base_dir) == str(get_path(base_dir, "config.default.yaml")) - - -def test_get_current_directory_path(): - """ - Verify the current directory path returned by get_current_directory_path() - """ - path = get_current_directory_path() - assert str(path) == os.getcwd() - - -def test_get_relative_path(get_temp_file): - """ - Verify the relative path returned by get_relative_path() - """ - path = get_temp_file - # path relative to the parent directory of the temp file - relative_path = get_relative_path(path, get_path(path).parent) - assert str(relative_path) == _name_temp_file - assert str(relative_path) == os.path.relpath(path, start=get_path(path).parent) - - -def test_two_2_three_digits_country(): - """ - Verify the conversion from two-digit to three-digit country code. - """ - # Afghanistan - assert two_2_three_digits_country("AF") == "AFG" - # American Samoa - assert two_2_three_digits_country("AS") == "ASM" - # Aruba - assert two_2_three_digits_country("AW") == "ABW" - # Germany - assert two_2_three_digits_country("DE") == "DEU" - # Micronesia (Federated States of) - assert two_2_three_digits_country("FM") == "FSM" - - -def test_three_2_two_digits_country(): - """ - Verify the conversion from three-digit to two-digit country code. - """ - # Afghanistan - assert "AF" == three_2_two_digits_country("AFG") - # American Samoa - assert "AS" == three_2_two_digits_country("ASM") - # Aruba - assert "AW" == three_2_two_digits_country("ABW") - # Germany - assert "DE" == three_2_two_digits_country("DEU") - # Micronesia (Federated States of) - assert "FM" == three_2_two_digits_country("FSM") - - -def test_two_digits_2_name_country(): - """ - Verify the conversion from two-digit country code to country name. - """ - # Micronesia (Federated States of) - assert "Micronesia, Fed. Sts." == two_digits_2_name_country("FM") - assert "Federated States of Micronesia" == two_digits_2_name_country( - "FM", name_string="name_official" - ) - assert "States of Micronesia" == two_digits_2_name_country( - "FM", name_string="name_official", remove_start_words=["Federated "] - ) - # Democratic Republic of the Congo - assert "DR Congo" == two_digits_2_name_country("CD") - assert "Democratic Republic of the Congo" == two_digits_2_name_country( - "CD", name_string="name_official" - ) - assert "Republic of the Congo" == two_digits_2_name_country( - "CD", name_string="name_official", remove_start_words=["Democratic "] - ) - - -def test_country_name_2_two_digits(): - """ - Verify the conversion from country name to two-digit country code. - """ - # Afghanistan - assert "AF" == country_name_2_two_digits("Afghanistan") - # American Samoa - assert "AS" == country_name_2_two_digits("American Samoa") - # Aruba - assert "AW" == country_name_2_two_digits("Aruba") - # Germany - assert "DE" == country_name_2_two_digits("Germany") - # Micronesia (Federated States of) - assert "FM" == country_name_2_two_digits("Micronesia") - - -def test_safe_divide(): - """ - Verify that the method safe_divide prevents divisions by vanishing - denominator. - """ - assert safe_divide(3.0, 2.0) == 1.5 - assert np.isnan(safe_divide(3.0, 0.0)) - - -def test_get_conv_factors(): - """ - Verify that the conversion factors returned by get_conv_factors are - correct. - """ - conversion_factors_dict = get_conv_factors("industry") - assert conversion_factors_dict["additives and oxygenates"] == 0.008333 - assert conversion_factors_dict["anthracite"] == 0.005 - assert conversion_factors_dict["aviation gasoline"] == 0.01230 - assert conversion_factors_dict["bagasse"] == 0.002144 - assert conversion_factors_dict["biodiesel"] == 0.01022 - assert conversion_factors_dict["biogasoline"] == 0.007444 - assert conversion_factors_dict["bio jet kerosene"] == 0.011111 - assert conversion_factors_dict["bitumen"] == 0.01117 - assert conversion_factors_dict["brown coal"] == 0.003889 - assert conversion_factors_dict["brown coal briquettes"] == 0.00575 - assert conversion_factors_dict["charcoal"] == 0.00819 - assert conversion_factors_dict["coal tar"] == 0.007778 - assert conversion_factors_dict["coke-oven coke"] == 0.0078334 - assert conversion_factors_dict["coke-oven gas"] == 0.000277 - assert conversion_factors_dict["coking coal"] == 0.007833 - assert conversion_factors_dict["conventional crude oil"] == 0.01175 - assert conversion_factors_dict["crude petroleum"] == 0.011750 - assert conversion_factors_dict["ethane"] == 0.01289 - assert conversion_factors_dict["fuel oil"] == 0.01122 - assert conversion_factors_dict["fuelwood"] == 0.00254 - assert conversion_factors_dict["gas coke"] == 0.007326 - assert conversion_factors_dict["gas oil/ diesel oil"] == 0.01194 - assert conversion_factors_dict["gasoline-type jet fuel"] == 0.01230 - assert conversion_factors_dict["hard coal"] == 0.007167 - assert conversion_factors_dict["kerosene-type jet fuel"] == 0.01225 - assert conversion_factors_dict["lignite"] == 0.003889 - assert conversion_factors_dict["liquefied petroleum gas (lpg)"] == 0.01313 - assert conversion_factors_dict["lubricants"] == 0.011166 - assert conversion_factors_dict["motor gasoline"] == 0.01230 - assert conversion_factors_dict["naphtha"] == 0.01236 - assert conversion_factors_dict["natural gas liquids"] == 0.01228 - assert conversion_factors_dict["oil shale"] == 0.00247 - assert conversion_factors_dict["other bituminous coal"] == 0.005556 - assert conversion_factors_dict["paraffin waxes"] == 0.01117 - assert conversion_factors_dict["patent fuel"] == 0.00575 - assert conversion_factors_dict["peat"] == 0.00271 - assert conversion_factors_dict["peat products"] == 0.00271 - assert conversion_factors_dict["petroleum coke"] == 0.009028 - assert conversion_factors_dict["refinery gas"] == 0.01375 - assert conversion_factors_dict["sub-bituminous coal"] == 0.005555 - assert np.isnan(get_conv_factors("non-industry")) - - -def test_modify_commodity(): - """ - Verify that modify_commodity returns the commodities in wished format. - """ - new_commodity_dataframe = pd.DataFrame() - new_commodity_dataframe["Commodity"] = ( - original_commodity_dataframe["Commodity"].map(modify_commodity).unique() - ) - df = new_commodity_dataframe.compare(modified_commodity_dataframe) - boolean_flag = df.empty - if not boolean_flag: - assert False - - -def test_aggregate_fuels(): - """ - Verify what is returned by aggregate_fuels. - """ - assert np.isnan(aggregate_fuels("non-industry")) - - -def test_get_gadm_filename(): - """ - Verify what is returned by get_gadm_filename. - """ - # Kosovo - assert get_gadm_filename("XK") == "gadm41_XKO" - # Clipperton island - assert get_gadm_filename("CP") == "gadm41_XCL" - # Saint-Martin - assert get_gadm_filename("SX") == "gadm41_MAF" - # French Southern Territories - assert get_gadm_filename("TF") == "gadm41_ATF" - # Aland - assert get_gadm_filename("AX") == "gadm41_ALA" - # British Indian Ocean Territory - assert get_gadm_filename("IO") == "gadm41_IOT" - # Cocos Islands - assert get_gadm_filename("CC") == "gadm41_CCK" - # Norfolk - assert get_gadm_filename("NF") == "gadm41_NFK" - # Pitcairn Islands - assert get_gadm_filename("PN") == "gadm41_PCN" - # Jersey - assert get_gadm_filename("JE") == "gadm41_JEY" - # Spratly Islands - assert get_gadm_filename("XS") == "gadm41_XSP" - # Guernsey - assert get_gadm_filename("GG") == "gadm41_GGY" - # United States Minor Outlying Islands - assert get_gadm_filename("UM") == "gadm41_UMI" - # Svalbard islands - assert get_gadm_filename("SJ") == "gadm41_SJM" - # Christmas island - assert get_gadm_filename("CX") == "gadm41_CXR" - # Afghanistan - assert get_gadm_filename("AF") == "gadm41_AFG" - # American Samoa - assert get_gadm_filename("AS") == "gadm41_ASM" - # Aruba - assert get_gadm_filename("AW") == "gadm41_ABW" - # Germany - assert get_gadm_filename("DE") == "gadm41_DEU" - # Micronesia (Federated States of) - assert get_gadm_filename("FM") == "gadm41_FSM" - # Micronesia (Federated States of) with different file_prefix - assert get_gadm_filename("FM", file_prefix="gadm456_") == "gadm456_FSM" - - -def test_get_gadm_url(): - """ - Verify what is returned by get_gadm_url. - """ - gadm_filename = get_gadm_filename("FM") - url_gadm41 = get_gadm_url( - "https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/", - gadm_filename, - ) - assert ( - url_gadm41 - == f"https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/{gadm_filename}.gpkg" - ) - - -def test_download_gadm(): - """ - Verify what is returned by download_gadm. - """ - file_prefix_41 = "gadm41_" - gadm_url_prefix_41 = "https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/" - gadm_input_file_args_41 = ["data", "gadm"] - gadm_input_file_gpkg_41, gadm_filename_41 = download_gadm( - "XK", - file_prefix_41, - gadm_url_prefix_41, - gadm_input_file_args_41, - update=True, - ) - assert gadm_input_file_gpkg_41 == get_path( - path_cwd, "data/gadm/gadm41_XKO/gadm41_XKO.gpkg" - ) - assert gadm_filename_41 == "gadm41_XKO" - list_layers_41 = fiona.listlayers(gadm_input_file_gpkg_41) - assert list_layers_41[0] == "ADM_ADM_0" - assert list_layers_41[1] == "ADM_ADM_1" - assert list_layers_41[2] == "ADM_ADM_2" - - -def test_normed(): - df_input = pd.DataFrame( - {"A": [1.0, 2.0, 3.0, 4.0, 5.0], "B": [6.0, 7.0, 8.0, 9.0, 10.0]} - ) - df_output = normed(df_input) - df_reference = pd.DataFrame( - { - "A": [x / 15.0 for x in [1.0, 2.0, 3.0, 4.0, 5.0]], - "B": [x / 40.0 for x in [6.0, 7.0, 8.0, 9.0, 10.0]], - } - ) - df_comparison = df_output.compare(df_reference) - assert df_comparison.empty diff --git a/test/test_prepare_network.py b/test/test_prepare_network.py deleted file mode 100644 index a5b4779d8..000000000 --- a/test/test_prepare_network.py +++ /dev/null @@ -1,233 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - - -import sys - -from pandas import Timestamp - -sys.path.append("./scripts") - -from test.conftest import get_power_network_ac_dc_meshed, get_power_network_scigrid_de - -from prepare_network import ( - add_co2limit, - add_emission_prices, - add_gaslimit, - average_every_nhours, - download_emission_data, - emission_extractor, - enforce_autarky, - set_line_nom_max, - set_line_s_max_pu, -) - -automatic_emission_base_year = 1990 -country_names = ["DE", "IT", "NG"] -co2limit = 7.75e7 - - -def test_download_emission_data(): - """ - Verify what returned by download_emission_data. - """ - file_name = download_emission_data() - assert file_name == "v60_CO2_excl_short-cycle_org_C_1970_2018.xls" - - -def test_emission_extractor(): - """ - Verify what returned by emission_extractor. - """ - output_series = emission_extractor( - "v60_CO2_excl_short-cycle_org_C_1970_2018.xls", - automatic_emission_base_year, - country_names, - ) - assert output_series.index.tolist() == ["NGA", "DEU", "ITA"] - assert output_series.values.tolist() == [ - 5698.76187, - 381475.887377666, - 123981.6458729, - ] - - -def test_add_co2limit(get_power_network_scigrid_de): - """ - Verify what returned by add_co2limit. - """ - test_network_de = get_power_network_scigrid_de - number_years = test_network_de.snapshot_weightings.objective.sum() / 8760.0 - add_co2limit(test_network_de, co2limit, number_years) - assert ( - test_network_de.global_constraints.carrier_attribute.values[0] - == "co2_emissions" - ) - assert test_network_de.global_constraints.sense.values[0] == "<=" - assert ( - test_network_de.global_constraints.constant.values[0] == co2limit * number_years - ) - - -def test_add_gaslimit(get_power_network_scigrid_de): - """ - Verify what returned by add_gaslimit. - """ - test_network_de = get_power_network_scigrid_de - test_network_de.add("Carrier", "OCGT") - number_years = test_network_de.snapshot_weightings.objective.sum() / 8760.0 - add_gaslimit(test_network_de, number_years, number_years) - assert test_network_de.global_constraints.carrier_attribute.values[0] == "gas_usage" - assert test_network_de.global_constraints.sense.values[0] == "<=" - assert ( - test_network_de.global_constraints.constant.values[0] - == number_years * number_years - ) - - -def test_add_emission_prices(get_power_network_ac_dc_meshed): - """ - Verify what returned by add_emission_prices. - """ - test_network_ac_dc_meshed = get_power_network_ac_dc_meshed - add_emission_prices( - test_network_ac_dc_meshed, emission_prices={"co2": 1.0}, exclude_co2=False - ) - assert test_network_ac_dc_meshed.generators["marginal_cost"].index.tolist() == [ - "Manchester Wind", - "Manchester Gas", - "Norway Wind", - "Norway Gas", - "Frankfurt Wind", - "Frankfurt Gas", - ] - assert test_network_ac_dc_meshed.generators["marginal_cost"].values.tolist() == [ - 0.11, - 5.218030132047726, - 0.09, - 6.565421697244602, - 0.1, - 4.768788024122113, - ] - - -def test_set_line_s_max_pu(get_power_network_scigrid_de): - """ - Verify what returned by set_line_s_max_pu. - """ - test_network_de = get_power_network_scigrid_de - s_max_pu_new_value = 3.0 - set_line_s_max_pu(test_network_de, s_max_pu_new_value) - assert test_network_de.lines["s_max_pu"].unique()[0] == s_max_pu_new_value - - -def test_average_every_nhours(get_power_network_scigrid_de): - """ - Verify what returned by average_every_nhours. - """ - test_network_de = get_power_network_scigrid_de - - # The input network is already sampled in 1H snapshots. - # Hence, average_every_nhours should not change anything - new_network_1h = average_every_nhours(test_network_de, "1H") - assert test_network_de.snapshots.tolist() == new_network_1h.snapshots.tolist() - - # Re-sample to 4H intervals - new_network_4h = average_every_nhours(test_network_de, "4H") - assert new_network_4h.snapshots.tolist() == [ - Timestamp("2011-01-01 00:00:00"), - Timestamp("2011-01-01 04:00:00"), - Timestamp("2011-01-01 08:00:00"), - Timestamp("2011-01-01 12:00:00"), - Timestamp("2011-01-01 16:00:00"), - Timestamp("2011-01-01 20:00:00"), - ] - - -def test_enforce_autarky_only_crossborder_false(get_power_network_ac_dc_meshed): - """ - Verify what returned by enforce_autarky when only_crossborder is False. - """ - # --> it removes all lines and all DC links - test_network_no_cross_border = get_power_network_ac_dc_meshed - - bus_country_list = ["UK", "UK", "UK", "UK", "DE", "DE", "DE", "NO", "NO"] - test_network_no_cross_border.buses["country"] = bus_country_list - test_network_no_cross_border.links["carrier"] = "DC" - - enforce_autarky(test_network_no_cross_border, only_crossborder=False) - - output_component_dict_no_cross_border = {} - for c in test_network_no_cross_border.iterate_components( - list(test_network_no_cross_border.components.keys())[2:] - ): - output_component_dict_no_cross_border[c.name] = len(c.df) - - reference_component_dict_no_cross_border = { - "Bus": 9, - "Carrier": 3, - "GlobalConstraint": 1, - "LineType": 34, - "TransformerType": 14, - "Load": 6, - "Generator": 6, - } - assert ( - output_component_dict_no_cross_border - == reference_component_dict_no_cross_border - ) - - -def test_enforce_autarky_only_crossborder_true(get_power_network_ac_dc_meshed): - """ - Verify what returned by enforce_autarky when only_crossborder is True. - """ - # --> it removes links and lines that cross borders - test_network_with_cross_border = get_power_network_ac_dc_meshed - bus_country_list = ["UK", "UK", "UK", "UK", "DE", "DE", "DE", "NO", "NO"] - test_network_with_cross_border.buses["country"] = bus_country_list - test_network_with_cross_border.links["carrier"] = "DC" - - enforce_autarky(test_network_with_cross_border, only_crossborder=True) - - output_component_dict_with_cross_border = {} - for c in test_network_with_cross_border.iterate_components( - list(test_network_with_cross_border.components.keys())[2:] - ): - output_component_dict_with_cross_border[c.name] = len(c.df) - - reference_component_dict_with_cross_border = { - "Bus": 9, - "Carrier": 3, - "GlobalConstraint": 1, - "Line": 4, - "LineType": 34, - "TransformerType": 14, - "Link": 3, - "Load": 6, - "Generator": 6, - } - print(output_component_dict_with_cross_border) - - assert ( - output_component_dict_with_cross_border - == reference_component_dict_with_cross_border - ) - - -def test_set_line_nom_max(get_power_network_ac_dc_meshed): - """ - Verify what returned by set_line_nom_max. - """ - test_network = get_power_network_ac_dc_meshed - s_nom_max_value = 5.0 - p_nom_max_value = 10.0 - set_line_nom_max( - test_network, s_nom_max_set=s_nom_max_value, p_nom_max_set=p_nom_max_value - ) - assert test_network.lines.s_nom_max.unique()[0] == s_nom_max_value - assert test_network.links.p_nom_max.unique()[0] == p_nom_max_value