From 747441f216019226efe26ec65570bdc1ef968b08 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:47:34 +0000 Subject: [PATCH 01/68] docs(contributor): contrib-readme-action has updated readme --- README.md | 167 ++++++++++++++++++++++++++---------------------------- 1 file changed, 80 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index c20cde919..c15921cab 100644 --- a/README.md +++ b/README.md @@ -189,17 +189,17 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe - - - - - - - - - + + + - + + + + + + + + - + + + + - + + + - + + + - + + + + +
- - FabianHofmann + + GbotemiB
- Fabian Hofmann + Emmanuel Bolarinwa 		- - fneum + + davide-f
- Fabian Neumann + Davide-f 		@@ -209,42 +209,6 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe Ekaterina - - euronion -
- Euronion - - 		- - Justus-coded -
- Justus Ilemobayo - - 		- - mnm-matin -
- Mnm-matin - -
- - martacki -
- Martha Frysztacki - - 		- - LukasFrankenQ -
- Lukas Franken - - 		pz-max @@ -252,61 +216,47 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe Max Parzen - - davide-f -
- Davide-f - - 		- - koen-vg -
- Koen Van Greevenbroek - - 		hazemakhalek
Hazem -
energyLS
EnergyLS -
- - AnasAlgarei + + DeniseGiub
- AnasAlgarei + DeniseGiub 		- - yerbol-akhmetov + + FabianHofmann
- Yerbol Akhmetov + Fabian Hofmann 		- - DeniseGiub + + yerbol-akhmetov
- DeniseGiub + Yerbol Akhmetov 		- - GbotemiB + + mnm-matin
- Emmanuel Bolarinwa + Mnm-matin 		@@ -315,15 +265,15 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Eddy Jalbout -
doneachh
Anton Achhammer -
Tomkourou @@ -345,6 +295,13 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe GridGrapher + + martacki +
+ Martha Frysztacki + + 		glenkiely-ieg @@ -388,6 +345,28 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe Ekaterina-Vo + + euronion +
+ Euronion + + 		+ + AnasAlgarei +
+ AnasAlgarei + +
+ + LukasFrankenQ +
+ Lukas Franken + + 		Tooblippe @@ -396,13 +375,12 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe - - carlosfv92 + + asolavi
- Carlos Fernandez + Null -
rajesh-ieg @@ -411,11 +389,26 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe - - asolavi + + koen-vg +
+ Koen Van Greevenbroek + + 		+ + danielelerede-oet
Null +
+ + carlosfv92 +
+ Carlos Fernandez + 		@@ -444,15 +437,15 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Juli-a-ko -
squoilin
Sylvain Quoilin -
siddharth-krishna @@ -487,15 +480,15 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Jess -
jarry7
Jarrad Wright -
HanaElattar From d6afd52401b4ab69776e5d51be20c539ffc3f997 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Tue, 5 Nov 2024 00:19:01 +0100 Subject: [PATCH 02/68] Update PyPSA & enable linopy (#1167) * Add a zenodo link to natura.tiff * Update environment * Revise structure definition for lines * Remove get_aggregation_strategies * Fix typo aggregation_strategies * Replace aggregategenerators with aggregateoneport * Add aggregation strategies as a parameter * Re-define aggregation strategies * Update aggregation strategies * Update aggregation strategies for lines * Update aggregation strategies for buses * Fix typo * Put aggregation strategies into a variable * Parametrize the aggregation strategies * Refactor update of the aggregation strategies * Clean-up the code * Revert "Add a zenodo link to natura.tiff" This reverts commit 77007598b436df510ec8ce6f29efa9d067341628. * Define an explicit clustering strategy for v_nom * Add a release note * Get glpk back * Specify v_nom for buses explicitly * Revert "Specify v_nom for buses explicitly" This reverts commit 20192e6b3e80a2fedbee398f8e892f776bb5b5cc. * Add a version restriction to the environment specification * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Adjust naming * Move the variable definition * Move the variable * Upgrade PyPSA version --------- Co-authored-by: Davide Fioriti Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Snakefile | 3 ++ doc/release_notes.rst | 2 + envs/environment.yaml | 3 +- scripts/_helpers.py | 10 +++++ scripts/base_network.py | 14 ------- scripts/build_osm_network.py | 25 ++++++++++++ scripts/cluster_network.py | 39 +++++++++++++------ scripts/simplify_network.py | 75 ++++++++++++++++++++++++------------ 8 files changed, 120 insertions(+), 51 deletions(-) diff --git a/Snakefile b/Snakefile index 218041c67..8088117b7 100644 --- a/Snakefile +++ b/Snakefile @@ -563,6 +563,7 @@ rule add_electricity: rule simplify_network: params: + aggregation_strategies=config["cluster_options"]["aggregation_strategies"], renewable=config["renewable"], geo_crs=config["crs"]["geo_crs"], cluster_options=config["cluster_options"], @@ -605,6 +606,7 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == True: rule cluster_network: params: + aggregation_strategies=config["cluster_options"]["aggregation_strategies"], build_shape_options=config["build_shape_options"], electricity=config["electricity"], costs=config["costs"], @@ -690,6 +692,7 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == False: rule cluster_network: params: + aggregation_strategies=config["cluster_options"]["aggregation_strategies"], build_shape_options=config["build_shape_options"], electricity=config["electricity"], costs=config["costs"], diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0d1b7c746..c084a7725 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -72,6 +72,8 @@ PyPSA-Earth 0.4.0 * Add an option to use csv format for custom demand imports. `PR #995 `__ +* Implement changes in processing network topology to use the updated PyPSA version. `PR #1065 `__ + **Minor Changes and bug-fixing** * Minor bug-fixing to run the cluster wildcard min `PR #1019 `__ diff --git a/envs/environment.yaml b/envs/environment.yaml index 7da885a73..dc0726ebe 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -12,7 +12,7 @@ dependencies: - pip - mamba # esp for windows build -- pypsa>=0.24, <0.25 +- pypsa>=0.25, <0.29 # - atlite>=0.2.4 # until https://github.com/PyPSA/atlite/issues/244 is not merged - dask - powerplantmatching @@ -27,6 +27,7 @@ dependencies: - memory_profiler - ruamel.yaml<=0.17.26 - pytables +- pyscipopt # added to compy with the quadratic objective requirement of the clustering script - lxml - numpy - pandas diff --git a/scripts/_helpers.py b/scripts/_helpers.py index ce97f6171..a106f7185 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -922,6 +922,16 @@ def get_last_commit_message(path): return last_commit_message +def update_config_dictionary( + config_dict, + parameter_key_to_fill="lines", + dict_to_use={"geometry": "first", "bounds": "first"}, +): + config_dict.setdefault(parameter_key_to_fill, {}) + config_dict[parameter_key_to_fill].update(dict_to_use) + return config_dict + + # PYPSA-EARTH-SEC diff --git a/scripts/base_network.py b/scripts/base_network.py index 65d640d44..e11ff83c6 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -523,20 +523,6 @@ def base_network( result_type="reduce", ) n.import_components_from_dataframe(lines_ac, "Line") - # The columns which names starts with "bus" are mixed up with the third-bus specification - # when executing additional_linkports() - lines_dc.drop( - labels=[ - "bus0_lon", - "bus0_lat", - "bus1_lon", - "bus1_lat", - "bus_0_coors", - "bus_1_coors", - ], - axis=1, - inplace=True, - ) n.import_components_from_dataframe(lines_dc, "Link") n.import_components_from_dataframe(transformers, "Transformer") diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 867262abc..d8584cf4e 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -24,6 +24,27 @@ logger = create_logger(__name__) +# Keep only a predefined set of columns, as otherwise conflicts are possible +# e.g. the columns which names starts with "bus" are mixed up with +# the third-bus specification when executing additional_linkports() +LINES_COLUMNS = [ + "line_id", + "circuits", + "tag_type", + "voltage", + "bus0", + "bus1", + "length", + "underground", + "under_construction", + "tag_frequency", + "dc", + "country", + "geometry", + "bounds", +] + + def line_endings_to_bus_conversion(lines): # Assign to every line a start and end point @@ -813,6 +834,7 @@ def built_network( countries_config, geo_crs, distance_crs, + lines_cols_standard, force_ac=False, ): logger.info("Stage 1/5: Read input data") @@ -877,6 +899,8 @@ def built_network( if not os.path.exists(outputs["lines"]): os.makedirs(os.path.dirname(outputs["lines"]), exist_ok=True) + lines = lines[lines_cols_standard] + to_csv_nafix(lines, outputs["lines"]) # Generate CSV to_csv_nafix(converters, outputs["converters"]) # Generate CSV to_csv_nafix(transformers, outputs["transformers"]) # Generate CSV @@ -912,5 +936,6 @@ def built_network( countries, geo_crs, distance_crs, + lines_cols_standard=LINES_COLUMNS, force_ac=force_ac, ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index eeaa2a98a..74d284fb7 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -134,6 +134,7 @@ configure_logging, create_logger, get_aggregation_strategies, + update_config_dictionary, update_p_nom_max, ) from add_electricity import load_costs @@ -575,9 +576,10 @@ def clustering_for_n_clusters( extended_link_costs=0, focus_weights=None, ): - bus_strategies, generator_strategies = get_aggregation_strategies( - aggregation_strategies - ) + line_strategies = aggregation_strategies.get("lines", dict()) + bus_strategies = aggregation_strategies.get("buses", dict()) + generator_strategies = aggregation_strategies.get("generators", dict()) + one_port_strategies = aggregation_strategies.get("one_ports", dict()) if not isinstance(custom_busmap, pd.Series): if alternative_clustering: @@ -603,12 +605,14 @@ def clustering_for_n_clusters( clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, + line_strategies=line_strategies, + bus_strategies=bus_strategies, generator_strategies=generator_strategies, + one_port_strategies=one_port_strategies, scale_link_capital_costs=False, ) @@ -727,14 +731,27 @@ def consense(x): ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.params.cluster_options.get( - "aggregation_strategies", {} + aggregation_strategies = snakemake.params.aggregation_strategies + + # Aggregation strategies must be set for all columns + update_config_dictionary( + config_dict=aggregation_strategies, + parameter_key_to_fill="lines", + dict_to_use={"v_nom": "first", "geometry": "first", "bounds": "first"}, + ) + update_config_dictionary( + config_dict=aggregation_strategies, + parameter_key_to_fill="buses", + dict_to_use={ + "v_nom": "first", + "lat": "mean", + "lon": "mean", + "tag_substation": "first", + "tag_area": "first", + "country": "first", + }, ) - # translate str entries of aggregation_strategies to pd.Series functions: - aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() - } + custom_busmap = False # snakemake.params.custom_busmap custom busmap is depreciated https://github.com/pypsa-meets-earth/pypsa-earth/pull/694 if custom_busmap: busmap = pd.read_csv( diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 92c3dd340..502cf1b9d 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -96,13 +96,12 @@ from _helpers import ( configure_logging, create_logger, - get_aggregation_strategies, + update_config_dictionary, update_p_nom_max, ) from add_electricity import load_costs from cluster_network import cluster_regions, clustering_for_n_clusters from pypsa.clustering.spatial import ( - aggregategenerators, aggregateoneport, busmap_by_stubs, get_clustering_from_busmap, @@ -276,11 +275,15 @@ def replace_components(n, c, df, pnl): _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) - _, generator_strategies = get_aggregation_strategies(aggregation_strategies) + generator_strategies = aggregation_strategies["generators"] carriers = set(n.generators.carrier) - set(exclude_carriers) - generators, generators_pnl = aggregategenerators( - n, busmap, carriers=carriers, custom_strategies=generator_strategies + generators, generators_pnl = aggregateoneport( + n, + busmap, + "Generator", + carriers=carriers, + custom_strategies=generator_strategies, ) replace_components(n, "Generator", generators, generators_pnl) @@ -588,19 +591,22 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): if not dist.empty: busmap.loc[buses_i] = dist.idxmin(1) - bus_strategies, generator_strategies = get_aggregation_strategies( - aggregation_strategies - ) + line_strategies = aggregation_strategies.get("lines", dict()) + bus_strategies = aggregation_strategies.get("buses", dict()) + generator_strategies = aggregation_strategies.get("generators", dict()) + one_port_strategies = aggregation_strategies.get("one_ports", dict()) clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, + line_strategies=line_strategies, + bus_strategies=bus_strategies, generator_strategies=generator_strategies, + one_port_strategies=one_port_strategies, scale_link_capital_costs=False, ) return clustering.network, busmap @@ -848,19 +854,22 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()): if (busmap.index == busmap).all(): return n, n.buses.index.to_series() - bus_strategies, generator_strategies = get_aggregation_strategies( - aggregation_strategies - ) + line_strategies = aggregation_strategies.get("lines", dict()) + bus_strategies = aggregation_strategies.get("buses", dict()) + generator_strategies = aggregation_strategies.get("generators", dict()) + one_port_strategies = aggregation_strategies.get("one_ports", dict()) clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, + line_strategies=line_strategies, + bus_strategies=bus_strategies, generator_strategies=generator_strategies, + one_port_strategies=one_port_strategies, scale_link_capital_costs=False, ) @@ -934,19 +943,22 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()): if (busmap.index == busmap).all(): return n, n.buses.index.to_series() - bus_strategies, generator_strategies = get_aggregation_strategies( - aggregation_strategies - ) + line_strategies = aggregation_strategies.get("lines", dict()) + bus_strategies = aggregation_strategies.get("buses", dict()) + generator_strategies = aggregation_strategies.get("generators", dict()) + one_port_strategies = aggregation_strategies.get("one_ports", dict()) clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, + line_strategies=line_strategies, + bus_strategies=bus_strategies, generator_strategies=generator_strategies, + one_port_strategies=one_port_strategies, scale_link_capital_costs=False, ) @@ -976,14 +988,27 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()): "exclude_carriers", [] ) hvdc_as_lines = snakemake.params.electricity["hvdc_as_lines"] - aggregation_strategies = snakemake.params.cluster_options.get( - "aggregation_strategies", {} + aggregation_strategies = snakemake.params.aggregation_strategies + + # Aggregation strategies must be set for all columns + update_config_dictionary( + config_dict=aggregation_strategies, + parameter_key_to_fill="lines", + dict_to_use={"v_nom": "first", "geometry": "first", "bounds": "first"}, ) - # translate str entries of aggregation_strategies to pd.Series functions: - aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() - } + update_config_dictionary( + config_dict=aggregation_strategies, + parameter_key_to_fill="buses", + dict_to_use={ + "v_nom": "first", + "lat": "mean", + "lon": "mean", + "tag_substation": "first", + "tag_area": "first", + "country": "first", + }, + ) + n, trafo_map = simplify_network_to_base_voltage(n, linetype, base_voltage) Nyears = n.snapshot_weightings.objective.sum() / 8760 @@ -1088,7 +1113,7 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()): solver_name, cluster_config.get("algorithm", "hac"), cluster_config.get("feature", None), - aggregation_strategies, + aggregation_strategies=aggregation_strategies, ) busmaps.append(cluster_map) From bb953250dd6c1b7c992b91ac2ebfa6feb0393dda Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Tue, 5 Nov 2024 21:34:24 +0100 Subject: [PATCH 03/68] Enable linopy - power model (#1169) * Add a zenodo link to natura.tiff * Update environment * Revise structure definition for lines * Remove get_aggregation_strategies * Fix typo aggregation_strategies * Replace aggregategenerators with aggregateoneport * Add aggregation strategies as a parameter * Re-define aggregation strategies * Update aggregation strategies * Update aggregation strategies for lines * Update aggregation strategies for buses * Fix typo * Put aggregation strategies into a variable * Parametrize the aggregation strategies * Refactor update of the aggregation strategies * Clean-up the code * Revert "Add a zenodo link to natura.tiff" This reverts commit 77007598b436df510ec8ce6f29efa9d067341628. * Define an explicit clustering strategy for v_nom * Add a release note * Get glpk back * Specify v_nom for buses explicitly * Revert "Specify v_nom for buses explicitly" This reverts commit 20192e6b3e80a2fedbee398f8e892f776bb5b5cc. * Add a version restriction to the environment specification * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Adjust naming * Move the variable definition * Move the variable * Upgrade PyPSA version * Update docstring * Fix imports duplication * Update imports * Update the carrier-capacity constraint * Add docstring * Update the equity constraint * Add docstring * Update BAU constraint * Update SAFE constraint * Add docstring * Update operational reserve margin constraint * Add docstring * Add an new argument to the RM constraint * Update the update of capacity constraints * Update adding an operational reserve margin constraint * Update docstring * Update battery constraint * Add docstring * Update a constraint related to a RES share * Fix usage of add_ERS_constraints * Update solving script * Update a solving run * Fix typos --------- Co-authored-by: Davide Fioriti Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- scripts/solve_network.py | 529 ++++++++++++++++++++++----------------- 1 file changed, 302 insertions(+), 227 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a9bbfbaa1..f52d2508b 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -52,15 +52,15 @@ linear optimal power flow (plus investment planning) is provided in the `documentation of PyPSA `_. -The optimization is based on the ``pyomo=False`` setting in the :func:`network.lopf` and :func:`pypsa.linopf.ilopf` function. -Additionally, some extra constraints specified in :mod:`prepare_network` are added. +The optimization is based on the :func:`network.optimize` function. +Additionally, some extra constraints specified in :mod:`prepare_network` and :mod:`solve_network` are added. Solving the network in multiple iterations is motivated through the dependence of transmission line capacities and impedances on values of corresponding flows. As lines are expanded their electrical parameters change, which renders the optimisation bilinear even if the power flow equations are linearized. To retain the computational advantage of continuous linear programming, a sequential linear programming technique is used, where in between iterations the line impedances are updated. -Details (and errors made through this heuristic) are discussed in the paper +Details (and errors introduced through this heuristic) are discussed in the paper - Fabian Neumann and Tom Brown. `Heuristics for Transmission Expansion Planning in Low-Carbon Energy System Models `_), *16th International Conference on the European Energy Market*, 2019. `arXiv:1907.10548 `_. @@ -86,23 +86,17 @@ import pandas as pd import pypsa from _helpers import configure_logging, create_logger, override_component_attrs +from linopy import merge from pypsa.descriptors import get_switchable_as_dense as get_as_dense -from pypsa.linopf import ( - define_constraints, - define_variables, - get_var, - ilopf, - join_exprs, - linexpr, - network_lopf, -) -from pypsa.linopt import define_constraints, get_var, join_exprs, linexpr +from pypsa.optimization.abstract import optimize_transmission_expansion_iteratively +from pypsa.optimization.optimize import optimize +from vresutils.benchmark import memory_logger logger = create_logger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) -def prepare_network(n, solve_opts): +def prepare_network(n, solve_opts, config): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -159,6 +153,25 @@ def prepare_network(n, solve_opts): def add_CCL_constraints(n, config): + """ + Add CCL (country & carrier limit) constraint to the network. + + Add minimum and maximum levels of generator nominal capacity per carrier + for individual countries. Opts and path for agg_p_nom_minmax.csv must be defined + in config.yaml. Default file is available at data/agg_p_nom_minmax.csv. + + Parameters + ---------- + n : pypsa.Network + config : dict + + Example + ------- + scenario: + opts: [Co2L-CCL-24H] + electricity: + agg_p_nom_limits: data/agg_p_nom_minmax.csv + """ agg_p_nom_limits = config["electricity"].get("agg_p_nom_limits") try: @@ -174,32 +187,57 @@ def add_CCL_constraints(n, config): ) gen_country = n.generators.bus.map(n.buses.country) - # cc means country and carrier - p_nom_per_cc = ( - pd.DataFrame( - { - "p_nom": linexpr((1, get_var(n, "Generator", "p_nom"))), - "country": gen_country, - "carrier": n.generators.carrier, - } + capacity_variable = n.model["Generator-p_nom"] + + lhs = [] + ext_carriers = n.generators.query("p_nom_extendable").carrier.unique() + for c in ext_carriers: + ext_carrier = n.generators.query("p_nom_extendable and carrier == @c") + country_grouper = ( + ext_carrier.bus.map(n.buses.country) + .rename_axis("Generator-ext") + .rename("country") ) - .dropna(subset=["p_nom"]) - .groupby(["country", "carrier"]) - .p_nom.apply(join_exprs) + ext_carrier_per_country = capacity_variable.loc[ + country_grouper.index + ].groupby_sum(country_grouper) + lhs.append(ext_carrier_per_country) + lhs = merge(lhs, dim=pd.Index(ext_carriers, name="carrier")) + + min_matrix = agg_p_nom_minmax["min"].to_xarray().unstack().reindex_like(lhs) + max_matrix = agg_p_nom_minmax["max"].to_xarray().unstack().reindex_like(lhs) + + n.model.add_constraints( + lhs >= min_matrix, name="agg_p_nom_min", mask=min_matrix.notnull() + ) + n.model.add_constraints( + lhs <= max_matrix, name="agg_p_nom_max", mask=max_matrix.notnull() ) - minimum = agg_p_nom_minmax["min"].dropna() - if not minimum.empty: - minconstraint = define_constraints( - n, p_nom_per_cc[minimum.index], ">=", minimum, "agg_p_nom", "min" - ) - maximum = agg_p_nom_minmax["max"].dropna() - if not maximum.empty: - maxconstraint = define_constraints( - n, p_nom_per_cc[maximum.index], "<=", maximum, "agg_p_nom", "max" - ) def add_EQ_constraints(n, o, scaling=1e-1): + """ + Add equity constraints to the network. + + Currently this is only implemented for the electricity sector only. + + Opts must be specified in the config.yaml. + + Parameters + ---------- + n : pypsa.Network + o : str + + Example + ------- + scenario: + opts: [Co2L-EQ0.7-24h] + + Require each country or node to on average produce a minimal share + of its total electricity consumption itself. Example: EQ0.7c demands each country + to produce on average at least 70% of its consumption; EQ0.7 demands + each node to produce on average at least 70% of its consumption. + """ float_regex = "[0-9]*\.?[0-9]+" level = float(re.findall(float_regex, o)[0]) if o[-1] == "c": @@ -220,99 +258,150 @@ def add_EQ_constraints(n, o, scaling=1e-1): ) inflow = inflow.reindex(load.index).fillna(0.0) rhs = scaling * (level * load - inflow) + dispatch_variable = n.model["Generator-p"] lhs_gen = ( - linexpr( - (n.snapshot_weightings.generators * scaling, get_var(n, "Generator", "p").T) - ) - .T.groupby(ggrouper, axis=1) - .apply(join_exprs) + (dispatch_variable * (n.snapshot_weightings.generators * scaling)) + .groupby(ggrouper.to_xarray()) + .sum() + .sum("snapshot") ) - lhs_spill = ( - linexpr( - ( - -n.snapshot_weightings.stores * scaling, - get_var(n, "StorageUnit", "spill").T, - ) + # TODO: double check that this is really needed, why do have to subtract the spillage + if not n.storage_units_t.inflow.empty: + spillage_variable = n.model["StorageUnit-spill"] + lhs_spill = ( + (spillage_variable * (-n.snapshot_weightings.stores * scaling)) + .groupby_sum(sgrouper) + .groupby(sgrouper.to_xarray()) + .sum() + .sum("snapshot") ) - .T.groupby(sgrouper, axis=1) - .apply(join_exprs) - ) - lhs_spill = lhs_spill.reindex(lhs_gen.index).fillna("") - lhs = lhs_gen + lhs_spill - define_constraints(n, lhs, ">=", rhs, "equity", "min") + lhs = lhs_gen + lhs_spill + else: + lhs = lhs_gen + n.model.add_constraints(lhs >= rhs, name="equity_min") def add_BAU_constraints(n, config): - ext_c = n.generators.query("p_nom_extendable").carrier.unique() - mincaps = pd.Series( - config["electricity"].get("BAU_mincapacities", {key: 0 for key in ext_c}) - ) - lhs = ( - linexpr((1, get_var(n, "Generator", "p_nom"))) - .groupby(n.generators.carrier) - .apply(join_exprs) - ) - define_constraints(n, lhs, ">=", mincaps[lhs.index], "Carrier", "bau_mincaps") - - maxcaps = pd.Series( - config["electricity"].get("BAU_maxcapacities", {key: np.inf for key in ext_c}) - ) - lhs = ( - linexpr((1, get_var(n, "Generator", "p_nom"))) - .groupby(n.generators.carrier) - .apply(join_exprs) - ) - define_constraints(n, lhs, "<=", maxcaps[lhs.index], "Carrier", "bau_maxcaps") + """ + Add a per-carrier minimal overall capacity. + + BAU_mincapacities and opts must be adjusted in the config.yaml. + + Parameters + ---------- + n : pypsa.Network + config : dict + + Example + ------- + scenario: + opts: [Co2L-BAU-24h] + electricity: + BAU_mincapacities: + solar: 0 + onwind: 0 + OCGT: 100000 + offwind-ac: 0 + offwind-dc: 0 + Which sets minimum expansion across all nodes e.g. in Europe to 100GW. + OCGT bus 1 + OCGT bus 2 + ... > 100000 + """ + mincaps = pd.Series(config["electricity"]["BAU_mincapacities"]) + p_nom = n.model["Generator-p_nom"] + ext_i = n.generators.query("p_nom_extendable") + ext_carrier_i = xr.DataArray(ext_i.carrier.rename_axis("Generator-ext")) + lhs = p_nom.groupby(ext_carrier_i).sum() + rhs = mincaps[lhs.indexes["carrier"]].rename_axis("carrier") + n.model.add_constraints(lhs >= rhs, name="bau_mincaps") def add_SAFE_constraints(n, config): - peakdemand = ( - 1.0 + config["electricity"]["SAFE_reservemargin"] - ) * n.loads_t.p_set.sum(axis=1).max() - conv_techs = config["plotting"]["conv_techs"] + """ + Add a capacity reserve margin of a certain fraction above the peak demand. + Renewable generators and storage do not contribute. Ignores network. + + Parameters + ---------- + n : pypsa.Network + config : dict + + Example + ------- + config.yaml requires to specify opts: + + scenario: + opts: [Co2L-SAFE-24h] + electricity: + SAFE_reservemargin: 0.1 + Which sets a reserve margin of 10% above the peak demand. + """ + peakdemand = n.loads_t.p_set.sum(axis=1).max() + margin = 1.0 + config["electricity"]["SAFE_reservemargin"] + reserve_margin = peakdemand * margin + conventional_carriers = config["electricity"]["conventional_carriers"] + ext_gens_i = n.generators.query( + "carrier in @conventional_carriers & p_nom_extendable" + ).index + capacity_variable = n.model["Generator-p_nom"] + p_nom = n.model["Generator-p_nom"].loc[ext_gens_i] + lhs = p_nom.sum() exist_conv_caps = n.generators.query( - "~p_nom_extendable & carrier in @conv_techs" + "~p_nom_extendable & carrier in @entional_carriers" ).p_nom.sum() - ext_gens_i = n.generators.query("carrier in @conv_techs & p_nom_extendable").index - lhs = linexpr((1, get_var(n, "Generator", "p_nom")[ext_gens_i])).sum() - rhs = peakdemand - exist_conv_caps - define_constraints(n, lhs, ">=", rhs, "Safe", "mintotalcap") + rhs = reserve_margin - exist_conv_caps + n.model.add_constraints(lhs >= rhs, name="safe_mintotalcap") -def add_operational_reserve_margin_constraint(n, config): +def add_operational_reserve_margin_constraint(n, sns, config): + """ + Build reserve margin constraints based on the formulation + as suggested in GenX + https://energy.mit.edu/wp-content/uploads/2017/10/Enhanced-Decision-Support-for-a-Changing-Electricity-Landscape.pdf + It implies that the reserve margin also accounts for optimal + dispatch of distributed energy resources (DERs) and demand response + which is a novel feature of GenX. + """ reserve_config = config["electricity"]["operational_reserve"] EPSILON_LOAD = reserve_config["epsilon_load"] EPSILON_VRES = reserve_config["epsilon_vres"] CONTINGENCY = reserve_config["contingency"] # Reserve Variables - reserve = get_var(n, "Generator", "r") - lhs = linexpr((1, reserve)).sum(1) + n.model.add_variables( + 0, np.inf, coords=[sns, n.generators.index], name="Generator-r" + ) + reserve = n.model["Generator-r"] + lhs = reserve.sum("Generator") # Share of extendable renewable capacities ext_i = n.generators.query("p_nom_extendable").index vres_i = n.generators_t.p_max_pu.columns if not ext_i.empty and not vres_i.empty: capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] - renewable_capacity_variables = get_var(n, "Generator", "p_nom")[ - vres_i.intersection(ext_i) - ] - lhs += linexpr( - (-EPSILON_VRES * capacity_factor, renewable_capacity_variables) - ).sum(1) + renewable_capacity_variables = ( + n.model["Generator-p_nom"] + .loc[vres_i.intersection(ext_i)] + .rename({"Generator-ext": "Generator"}) + ) + lhs = merge( + lhs, + (renewable_capacity_variables * (-EPSILON_VRES * capacity_factor)).sum( + ["Generator"] + ), + ) - # Total demand at t - demand = n.loads_t.p.sum(1) + # Total demand per t + demand = get_as_dense(n, "Load", "p_set").sum(axis=1) # VRES potential of non extendable generators capacity_factor = n.generators_t.p_max_pu[vres_i.difference(ext_i)] renewable_capacity = n.generators.p_nom[vres_i.difference(ext_i)] - potential = (capacity_factor * renewable_capacity).sum(1) + potential = (capacity_factor * renewable_capacity).sum(axis=1) # Right-hand-side rhs = EPSILON_LOAD * demand + EPSILON_VRES * potential + CONTINGENCY - define_constraints(n, lhs, ">=", rhs, "Reserve margin") + n.model.add_constraints(lhs >= rhs, name="reserve_margin") def update_capacity_constraint(n): @@ -320,65 +409,84 @@ def update_capacity_constraint(n): ext_i = n.generators.query("p_nom_extendable").index fix_i = n.generators.query("not p_nom_extendable").index - dispatch = get_var(n, "Generator", "p") - reserve = get_var(n, "Generator", "r") + dispatch = n.model["Generator-p"] + reserve = n.model["Generator-r"] capacity_fixed = n.generators.p_nom[fix_i] p_max_pu = get_as_dense(n, "Generator", "p_max_pu") - lhs = linexpr((1, dispatch), (1, reserve)) + lhs = merge( + dispatch * 1, + reserve * 1, + ) if not ext_i.empty: - capacity_variable = get_var(n, "Generator", "p_nom") - lhs += linexpr((-p_max_pu[ext_i], capacity_variable)).reindex( - columns=gen_i, fill_value="" - ) + capacity_variable = n.model["Generator-p_nom"] + lhs = dispatch + reserve - capacity_variable * xr.DataArray(p_max_pu[ext_i]) rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0) - define_constraints(n, lhs, "<=", rhs, "Generators", "updated_capacity_constraint") + n.model.add_constraints( + lhs <= rhs, name="gen_updated_capacity_constraint", mask=rhs.notnull() + ) def add_operational_reserve_margin(n, sns, config): """ - Build reserve margin constraints based on the formulation given in - https://genxproject.github.io/GenX/dev/core/#Reserves. + Parameters + ---------- + n : pypsa.Network + sns: pd.DatetimeIndex + config : dict + + Example: + -------- + config.yaml requires to specify operational_reserve: + operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves + activate: true + epsilon_load: 0.02 # percentage of load at each snapshot + epsilon_vres: 0.02 # percentage of VRES at each snapshot + contingency: 400000 # MW """ - define_variables(n, 0, np.inf, "Generator", "r", axes=[sns, n.generators.index]) - - add_operational_reserve_margin_constraint(n, config) + add_operational_reserve_margin_constraint(n, sns, config) update_capacity_constraint(n) def add_battery_constraints(n): + """ + Add constraint ensuring that charger = discharger, i.e. + 1 * charger_size - efficiency * discharger_size = 0 + """ nodes = n.buses.index[n.buses.carrier == "battery"] - if nodes.empty or ("Link", "p_nom") not in n.variables.index: + # TODO Check if the second part of the condition can make sense + # if nodes.empty or ("Link", "p_nom") not in n.variables.index: + if nodes.empty: return - link_p_nom = get_var(n, "Link", "p_nom") - lhs = linexpr( - (1, link_p_nom[nodes + " charger"]), - ( - -n.links.loc[nodes + " discharger", "efficiency"].values, - link_p_nom[nodes + " discharger"].values, - ), + vars_link = n.model["Link-p_nom"] + eff = n.links.loc[nodes + " discharger", "efficiency"] + lhs = merge( + vars_link.sel({"Link-ext": nodes + " charger"}) * 1, + # for some reasons, eff is one element longer as compared with vars_link + vars_link.sel({"Link-ext": nodes + " discharger"}) * -eff[0], ) - define_constraints(n, lhs, "=", 0, "Link", "charger_ratio") + n.model.add_constraints(lhs == 0, name="link_charger_ratio") -def add_RES_constraints(n, res_share): +def add_RES_constraints(n, res_share, config): lgrouper = n.loads.bus.map(n.buses.country) + # TODO drop load ggrouper = n.generators.bus.map(n.buses.country) sgrouper = n.storage_units.bus.map(n.buses.country) cgrouper = n.links.bus0.map(n.buses.country) logger.warning( - "The add_RES_constraints functionality is still work in progress. " + "The add_RES_constraints() is still work in progress. " "Unexpected results might be incurred, particularly if " "temporal clustering is applied or if an unexpected change of technologies " - "is subject to the obtimisation." + "is subject to future improvements." ) load = ( @@ -388,103 +496,68 @@ def add_RES_constraints(n, res_share): rhs = res_share * load - res_techs = [ - "solar", - "onwind", - "offwind-dc", - "offwind-ac", - "battery", - "hydro", - "ror", - ] + renew_techs = config["electricity"]["renewable_carriers"] + charger = ["H2 electrolysis", "battery charger"] discharger = ["H2 fuel cell", "battery discharger"] - gens_i = n.generators.query("carrier in @res_techs").index - stores_i = n.storage_units.query("carrier in @res_techs").index + gens_i = n.generators.query("carrier in @renew_techs").index + stores_i = n.storage_units.query("carrier in @renew_techs").index + charger_i = n.links.query("carrier in @charger").index discharger_i = n.links.query("carrier in @discharger").index + stores_t_weights = n.snapshot_weightings.stores + # Generators + # TODO restore grouping by countries un-commenting calls of groupby() lhs_gen = ( - linexpr( - (n.snapshot_weightings.generators, get_var(n, "Generator", "p")[gens_i].T) - ) - .T.groupby(ggrouper, axis=1) - .apply(join_exprs) + (n.model["Generator-p"].loc[:, gens_i] * n.snapshot_weightings.generators) + # .groupby(ggrouper.to_xarray()) + .sum() ) # StorageUnits - lhs_dispatch = ( - ( - linexpr( - ( - n.snapshot_weightings.stores, - get_var(n, "StorageUnit", "p_dispatch")[stores_i].T, - ) - ) - .T.groupby(sgrouper, axis=1) - .apply(join_exprs) - ) - .reindex(lhs_gen.index) - .fillna("") + store_disp_expr = ( + n.model["StorageUnit-p_dispatch"].loc[:, stores_i] * stores_t_weights + ) + store_expr = n.model["StorageUnit-p_store"].loc[:, stores_i] * stores_t_weights + charge_expr = n.model["Link-p"].loc[:, charger_i] * stores_t_weights.apply( + lambda r: r * n.links.loc[charger_i].efficiency + ) + discharge_expr = n.model["Link-p"].loc[:, discharger_i] * stores_t_weights.apply( + lambda r: r * n.links.loc[discharger_i].efficiency ) + lhs_dispatch = ( + store_disp_expr + # .groupby(sgrouper) + .sum() + ) lhs_store = ( - ( - linexpr( - ( - -n.snapshot_weightings.stores, - get_var(n, "StorageUnit", "p_store")[stores_i].T, - ) - ) - .T.groupby(sgrouper, axis=1) - .apply(join_exprs) - ) - .reindex(lhs_gen.index) - .fillna("") + store_expr + # .groupby(sgrouper) + .sum() ) # Stores (or their resp. Link components) # Note that the variables "p0" and "p1" currently do not exist. # Thus, p0 and p1 must be derived from "p" (which exists), taking into account the link efficiency. lhs_charge = ( - ( - linexpr( - ( - -n.snapshot_weightings.stores, - get_var(n, "Link", "p")[charger_i].T, - ) - ) - .T.groupby(cgrouper, axis=1) - .apply(join_exprs) - ) - .reindex(lhs_gen.index) - .fillna("") + charge_expr + # .groupby(cgrouper) + .sum() ) lhs_discharge = ( - ( - linexpr( - ( - n.snapshot_weightings.stores.apply( - lambda r: r * n.links.loc[discharger_i].efficiency - ), - get_var(n, "Link", "p")[discharger_i], - ) - ) - .groupby(cgrouper, axis=1) - .apply(join_exprs) - ) - .reindex(lhs_gen.index) - .fillna("") + discharge_expr + # .groupby(cgrouper) + .sum() ) - # signs of resp. terms are coded in the linexpr. - # todo: for links (lhs_charge and lhs_discharge), account for snapshot weightings - lhs = lhs_gen + lhs_dispatch + lhs_store + lhs_charge + lhs_discharge + lhs = lhs_gen + lhs_dispatch - lhs_store - lhs_charge + lhs_discharge - define_constraints(n, lhs, "=", rhs, "RES share") + n.model.add_constraints(lhs == rhs, name="res_share") def add_land_use_constraint(n): @@ -876,7 +949,7 @@ def extra_functionality(n, snapshots): for o in opts: if "RES" in o: res_share = float(re.findall("[0-9]*\.?[0-9]+$", o)[0]) - add_RES_constraints(n, res_share) + add_RES_constraints(n, res_share, config) for o in opts: if "EQ" in o: add_EQ_constraints(n, o) @@ -927,40 +1000,44 @@ def extra_functionality(n, snapshots): add_co2_sequestration_limit(n, snapshots) -def solve_network(n, config, solving={}, opts="", **kwargs): +def solve_network(n, config, solving, **kwargs): set_of_options = solving["solver"]["options"] cf_solving = solving["options"] - solver_options = solving["solver_options"][set_of_options] if set_of_options else {} - solver_name = solving["solver"]["name"] + kwargs["solver_options"] = ( + solving["solver_options"][set_of_options] if set_of_options else {} + ) + kwargs["solver_name"] = solving["solver"]["name"] - track_iterations = cf_solving.get("track_iterations", False) - min_iterations = cf_solving.get("min_iterations", 4) - max_iterations = cf_solving.get("max_iterations", 6) + skip_iterations = cf_solving.get("skip_iterations", False) + if not n.lines.s_nom_extendable.any(): + skip_iterations = True + logger.info("No expandable lines found. Skipping iterative solving.") # add to network for extra_functionality n.config = config n.opts = opts - if cf_solving.get("skip_iterations", False): - network_lopf( - n, - solver_name=solver_name, - solver_options=solver_options, - extra_functionality=extra_functionality, - **kwargs, - ) + if skip_iterations: + status, condition = n.optimize(**kwargs) else: - ilopf( - n, - solver_name=solver_name, - solver_options=solver_options, - track_iterations=track_iterations, - min_iterations=min_iterations, - max_iterations=max_iterations, - extra_functionality=extra_functionality, - **kwargs, + kwargs["track_iterations"] = (cf_solving.get("track_iterations", False),) + kwargs["min_iterations"] = (cf_solving.get("min_iterations", 4),) + kwargs["max_iterations"] = (cf_solving.get("max_iterations", 6),) + status, condition = n.optimize.optimize_transmission_expansion_iteratively( + **kwargs ) + + if status != "ok": # and not rolling_horizon: + logger.warning( + f"Solving status '{status}' with termination condition '{condition}'" + ) + if "infeasible" in condition: + labels = n.model.compute_infeasibilities() + logger.info(f"Labels:\n{labels}") + n.model.print_infeasibilities() + raise RuntimeError("Solving status 'infeasible'") + return n @@ -978,11 +1055,8 @@ def solve_network(n, config, solving={}, opts="", **kwargs): configure_logging(snakemake) - tmpdir = snakemake.params.solving.get("tmpdir") - if tmpdir is not None: - Path(tmpdir).mkdir(parents=True, exist_ok=True) opts = snakemake.wildcards.opts.split("-") - solving = snakemake.params.solving + solve_opts = snakemake.config["solving"]["options"] is_sector_coupled = "sopts" in snakemake.wildcards.keys() @@ -992,10 +1066,11 @@ def solve_network(n, config, solving={}, opts="", **kwargs): else: n = pypsa.Network(snakemake.input.network) - if snakemake.params.augmented_line_connection.get("add_to_snakefile"): - n.lines.loc[n.lines.index.str.contains("new"), "s_nom_min"] = ( - snakemake.params.augmented_line_connection.get("min_expansion") - ) + # TODO Double-check handling the augmented case + # if snakemake.params.augmented_line_connection.get("add_to_snakefile"): + # n.lines.loc[n.lines.index.str.contains("new"), "s_nom_min"] = ( + # snakemake.params.augmented_line_connection.get("min_expansion") + # ) if ( snakemake.config["custom_data"]["add_existing"] @@ -1016,15 +1091,15 @@ def solve_network(n, config, solving={}, opts="", **kwargs): else: n_ref = None - n = prepare_network(n, solving["options"]) + # needed to get `n.model` property + n.optimize.create_model() + n = prepare_network(n, solve_opts, config=solve_opts) n = solve_network( n, config=snakemake.config, - solving=solving, - opts=opts, - solver_dir=tmpdir, - solver_logfile=snakemake.log.solver, + solving=snakemake.params.solving, + log_fn=snakemake.log.solver, ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) n.export_to_netcdf(snakemake.output[0]) From 5c48d605bcd8f529e1804b8a639770f4aeba3aac Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 5 Nov 2024 21:42:59 +0100 Subject: [PATCH 04/68] Get back CI for Windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5887ac014..08f69a702 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: os: - ubuntu-latest - macos-latest - # - windows-latest + - windows-latest runs-on: ${{ matrix.os }} From 35517e96a03470195afacf046a17bfd06bf68e54 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 5 Nov 2024 21:43:38 +0100 Subject: [PATCH 05/68] Update environment --- envs/environment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index dc0726ebe..e263e7a78 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -78,7 +78,6 @@ dependencies: # Default solver for tests (required for CI) - glpk -- ipopt - gurobi - pip: From c71c538b41b04d395a96c49b3e6a930fc2f79347 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 15 Oct 2024 07:40:45 +0000 Subject: [PATCH 06/68] Add H2 carrier if not exists --- scripts/prepare_sector_network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 40b3400bf..786d546bb 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -245,7 +245,8 @@ def H2_liquid_fossil_conversions(n, costs): def add_hydrogen(n, costs): "function to add hydrogen as an energy carrier with its conversion technologies from and to AC" - n.add("Carrier", "H2") + if not "H2" in n.carriers.index: + n.add("Carrier", "H2") n.madd( "Bus", From b40e643060e5a9253bf5b89cac8a6fb03505f4e3 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 15 Oct 2024 07:58:05 +0000 Subject: [PATCH 07/68] Add battery carrier if not exists only --- scripts/prepare_sector_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 786d546bb..6cb4be6ed 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1089,7 +1089,9 @@ def add_aviation(n, cost): def add_storage(n, costs): "function to add the different types of storage systems" - n.add("Carrier", "battery") + + if not "battery" in n.carriers.index: + n.add("Carrier", "battery") n.madd( "Bus", From fc11c023aea47b3078cdbd4041defcd9c5099cb1 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 22 Oct 2024 19:04:21 +0000 Subject: [PATCH 08/68] use H2 in both AC and DC buses in h2_hc_conversions function --- scripts/prepare_sector_network.py | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6cb4be6ed..a00d224fe 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1180,11 +1180,11 @@ def h2_hc_conversions(n, costs): if snakemake.config["sector"]["hydrogen"]["hydrogen_colors"]: n.madd( "Bus", - nodes + " blue H2", - location=nodes, + spatial.nodes + " blue H2", + location=spatial.nodes, carrier="blue H2", - x=n.buses.loc[list(nodes)].x.values, - y=n.buses.loc[list(nodes)].y.values, + x=n.buses.loc[list(spatial.nodes)].x.values, + y=n.buses.loc[list(spatial.nodes)].y.values, ) n.madd( @@ -1192,7 +1192,7 @@ def h2_hc_conversions(n, costs): spatial.nodes, suffix=" SMR CC", bus0=spatial.gas.nodes, - bus1=nodes + " blue H2", + bus1=spatial.nodes + " blue H2", bus2="co2 atmosphere", bus3=spatial.co2.nodes, p_nom_extendable=True, @@ -1207,9 +1207,9 @@ def h2_hc_conversions(n, costs): n.madd( "Link", - nodes + " blue H2", - bus0=nodes + " blue H2", - bus1=nodes + " H2", + spatial.nodes + " blue H2", + bus0=spatial.nodes + " blue H2", + bus1=spatial.nodes + " H2", carrier="blue H2", capital_cost=0, p_nom_extendable=True, @@ -1222,7 +1222,7 @@ def h2_hc_conversions(n, costs): spatial.nodes, suffix=" SMR CC", bus0=spatial.gas.nodes, - bus1=nodes + " H2", + bus1=spatial.nodes + " H2", bus2="co2 atmosphere", bus3=spatial.co2.nodes, p_nom_extendable=True, @@ -1239,18 +1239,18 @@ def h2_hc_conversions(n, costs): if snakemake.config["sector"]["hydrogen"]["hydrogen_colors"]: n.madd( "Bus", - nodes + " grey H2", - location=nodes, + spatial.nodes + " grey H2", + location=spatial.nodes, carrier="grey H2", - x=n.buses.loc[list(nodes)].x.values, - y=n.buses.loc[list(nodes)].y.values, + x=n.buses.loc[list(spatial.nodes)].x.values, + y=n.buses.loc[list(spatial.nodes)].y.values, ) n.madd( "Link", - nodes + " SMR", + spatial.nodes + " SMR", bus0=spatial.gas.nodes, - bus1=nodes + " grey H2", + bus1=spatial.nodes + " grey H2", bus2="co2 atmosphere", p_nom_extendable=True, carrier="SMR", @@ -1262,9 +1262,9 @@ def h2_hc_conversions(n, costs): n.madd( "Link", - nodes + " grey H2", - bus0=nodes + " grey H2", - bus1=nodes + " H2", + spatial.nodes + " grey H2", + bus0=spatial.nodes + " grey H2", + bus1=spatial.nodes + " H2", carrier="grey H2", capital_cost=0, p_nom_extendable=True, @@ -1274,9 +1274,9 @@ def h2_hc_conversions(n, costs): else: n.madd( "Link", - nodes + " SMR", + spatial.nodes + " SMR", bus0=spatial.gas.nodes, - bus1=nodes + " H2", + bus1=spatial.nodes + " H2", bus2="co2 atmosphere", p_nom_extendable=True, carrier="SMR", From 38f6090a1b6b3d925b97b96371ebabcd0fa53805 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 22 Oct 2024 19:22:54 +0000 Subject: [PATCH 09/68] use AC and DC based H2 to make parameters of m.add consistent shape --- scripts/prepare_sector_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index a00d224fe..acb6394bb 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1344,7 +1344,7 @@ def add_shipping(n, costs): if options["shipping_hydrogen_liquefaction"]: n.madd( "Bus", - nodes, + spatial.nodes, suffix=" H2 liquid", carrier="H2 liquid", location=spatial.nodes, @@ -1373,7 +1373,7 @@ def add_shipping(n, costs): ): n.madd( "Load", - nodes, + spatial.nodes, suffix=" H2 for shipping", bus=shipping_bus, carrier="H2 for shipping", From 8c9077eb1ac395663d0d599c2ceffb03a9fb33ff Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Wed, 30 Oct 2024 19:40:38 +0500 Subject: [PATCH 10/68] add release notes --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 5b24e23d8..0d1b7c746 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -40,6 +40,7 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * The computation of `hydro_profile.nc` in `build_renewable_profiles.py` is not differentiated whether alternative clustering is applied or not; the indexing of the different power plants in `add_electricity.py` is performed according to the bus either in case alternative clustering is applied or not and a `hydro_inflow_factor` is computed prior to the computation of `inflow_t` to split the inflow according to the capacity of each different unit of each power plant (if more units are present). `PR #1119 `_ +* Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `_ PyPSA-Earth 0.4.1 ================= From 5f52c502be7b353886988b411e0e379b957e8c13 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 00:17:54 +0100 Subject: [PATCH 11/68] docs(contributor): contrib-readme-action has updated readme (#1162) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c15921cab..654c81026 100644 --- a/README.md +++ b/README.md @@ -388,13 +388,6 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe Null - - koen-vg -
- Koen Van Greevenbroek - - 		danielelerede-oet @@ -403,6 +396,13 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
+ + carlosfv92 +
+ Carlos Fernandez + + 		carlosfv92 @@ -430,7 +430,8 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Null -
juli-a-ko @@ -473,7 +474,8 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Katherine M. Antonio -
jessLryan From 763fb321220201bd8a27cead5e1badca473f8abf Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Tue, 5 Nov 2024 00:19:01 +0100 Subject: [PATCH 12/68] Update PyPSA & enable linopy (#1167) * Add a zenodo link to natura.tiff * Update environment * Revise structure definition for lines * Remove get_aggregation_strategies * Fix typo aggregation_strategies * Replace aggregategenerators with aggregateoneport * Add aggregation strategies as a parameter * Re-define aggregation strategies * Update aggregation strategies * Update aggregation strategies for lines * Update aggregation strategies for buses * Fix typo * Put aggregation strategies into a variable * Parametrize the aggregation strategies * Refactor update of the aggregation strategies * Clean-up the code * Revert "Add a zenodo link to natura.tiff" This reverts commit 77007598b436df510ec8ce6f29efa9d067341628. * Define an explicit clustering strategy for v_nom * Add a release note * Get glpk back * Specify v_nom for buses explicitly * Revert "Specify v_nom for buses explicitly" This reverts commit 20192e6b3e80a2fedbee398f8e892f776bb5b5cc. * Add a version restriction to the environment specification * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Adjust naming * Move the variable definition * Move the variable * Upgrade PyPSA version --------- Co-authored-by: Davide Fioriti Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Snakefile | 3 ++ doc/release_notes.rst | 2 + envs/environment.yaml | 3 +- scripts/_helpers.py | 10 +++++ scripts/base_network.py | 14 ------- scripts/build_osm_network.py | 25 ++++++++++++ scripts/cluster_network.py | 39 +++++++++++++------ scripts/simplify_network.py | 75 ++++++++++++++++++++++++------------ 8 files changed, 120 insertions(+), 51 deletions(-) diff --git a/Snakefile b/Snakefile index 218041c67..8088117b7 100644 --- a/Snakefile +++ b/Snakefile @@ -563,6 +563,7 @@ rule add_electricity: rule simplify_network: params: + aggregation_strategies=config["cluster_options"]["aggregation_strategies"], renewable=config["renewable"], geo_crs=config["crs"]["geo_crs"], cluster_options=config["cluster_options"], @@ -605,6 +606,7 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == True: rule cluster_network: params: + aggregation_strategies=config["cluster_options"]["aggregation_strategies"], build_shape_options=config["build_shape_options"], electricity=config["electricity"], costs=config["costs"], @@ -690,6 +692,7 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == False: rule cluster_network: params: + aggregation_strategies=config["cluster_options"]["aggregation_strategies"], build_shape_options=config["build_shape_options"], electricity=config["electricity"], costs=config["costs"], diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0d1b7c746..c084a7725 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -72,6 +72,8 @@ PyPSA-Earth 0.4.0 * Add an option to use csv format for custom demand imports. `PR #995 `__ +* Implement changes in processing network topology to use the updated PyPSA version. `PR #1065 `__ + **Minor Changes and bug-fixing** * Minor bug-fixing to run the cluster wildcard min `PR #1019 `__ diff --git a/envs/environment.yaml b/envs/environment.yaml index 7da885a73..dc0726ebe 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -12,7 +12,7 @@ dependencies: - pip - mamba # esp for windows build -- pypsa>=0.24, <0.25 +- pypsa>=0.25, <0.29 # - atlite>=0.2.4 # until https://github.com/PyPSA/atlite/issues/244 is not merged - dask - powerplantmatching @@ -27,6 +27,7 @@ dependencies: - memory_profiler - ruamel.yaml<=0.17.26 - pytables +- pyscipopt # added to compy with the quadratic objective requirement of the clustering script - lxml - numpy - pandas diff --git a/scripts/_helpers.py b/scripts/_helpers.py index ce97f6171..a106f7185 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -922,6 +922,16 @@ def get_last_commit_message(path): return last_commit_message +def update_config_dictionary( + config_dict, + parameter_key_to_fill="lines", + dict_to_use={"geometry": "first", "bounds": "first"}, +): + config_dict.setdefault(parameter_key_to_fill, {}) + config_dict[parameter_key_to_fill].update(dict_to_use) + return config_dict + + # PYPSA-EARTH-SEC diff --git a/scripts/base_network.py b/scripts/base_network.py index 65d640d44..e11ff83c6 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -523,20 +523,6 @@ def base_network( result_type="reduce", ) n.import_components_from_dataframe(lines_ac, "Line") - # The columns which names starts with "bus" are mixed up with the third-bus specification - # when executing additional_linkports() - lines_dc.drop( - labels=[ - "bus0_lon", - "bus0_lat", - "bus1_lon", - "bus1_lat", - "bus_0_coors", - "bus_1_coors", - ], - axis=1, - inplace=True, - ) n.import_components_from_dataframe(lines_dc, "Link") n.import_components_from_dataframe(transformers, "Transformer") diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 867262abc..d8584cf4e 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -24,6 +24,27 @@ logger = create_logger(__name__) +# Keep only a predefined set of columns, as otherwise conflicts are possible +# e.g. the columns which names starts with "bus" are mixed up with +# the third-bus specification when executing additional_linkports() +LINES_COLUMNS = [ + "line_id", + "circuits", + "tag_type", + "voltage", + "bus0", + "bus1", + "length", + "underground", + "under_construction", + "tag_frequency", + "dc", + "country", + "geometry", + "bounds", +] + + def line_endings_to_bus_conversion(lines): # Assign to every line a start and end point @@ -813,6 +834,7 @@ def built_network( countries_config, geo_crs, distance_crs, + lines_cols_standard, force_ac=False, ): logger.info("Stage 1/5: Read input data") @@ -877,6 +899,8 @@ def built_network( if not os.path.exists(outputs["lines"]): os.makedirs(os.path.dirname(outputs["lines"]), exist_ok=True) + lines = lines[lines_cols_standard] + to_csv_nafix(lines, outputs["lines"]) # Generate CSV to_csv_nafix(converters, outputs["converters"]) # Generate CSV to_csv_nafix(transformers, outputs["transformers"]) # Generate CSV @@ -912,5 +936,6 @@ def built_network( countries, geo_crs, distance_crs, + lines_cols_standard=LINES_COLUMNS, force_ac=force_ac, ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index eeaa2a98a..74d284fb7 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -134,6 +134,7 @@ configure_logging, create_logger, get_aggregation_strategies, + update_config_dictionary, update_p_nom_max, ) from add_electricity import load_costs @@ -575,9 +576,10 @@ def clustering_for_n_clusters( extended_link_costs=0, focus_weights=None, ): - bus_strategies, generator_strategies = get_aggregation_strategies( - aggregation_strategies - ) + line_strategies = aggregation_strategies.get("lines", dict()) + bus_strategies = aggregation_strategies.get("buses", dict()) + generator_strategies = aggregation_strategies.get("generators", dict()) + one_port_strategies = aggregation_strategies.get("one_ports", dict()) if not isinstance(custom_busmap, pd.Series): if alternative_clustering: @@ -603,12 +605,14 @@ def clustering_for_n_clusters( clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=aggregate_carriers, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=line_length_factor, + line_strategies=line_strategies, + bus_strategies=bus_strategies, generator_strategies=generator_strategies, + one_port_strategies=one_port_strategies, scale_link_capital_costs=False, ) @@ -727,14 +731,27 @@ def consense(x): ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.params.cluster_options.get( - "aggregation_strategies", {} + aggregation_strategies = snakemake.params.aggregation_strategies + + # Aggregation strategies must be set for all columns + update_config_dictionary( + config_dict=aggregation_strategies, + parameter_key_to_fill="lines", + dict_to_use={"v_nom": "first", "geometry": "first", "bounds": "first"}, + ) + update_config_dictionary( + config_dict=aggregation_strategies, + parameter_key_to_fill="buses", + dict_to_use={ + "v_nom": "first", + "lat": "mean", + "lon": "mean", + "tag_substation": "first", + "tag_area": "first", + "country": "first", + }, ) - # translate str entries of aggregation_strategies to pd.Series functions: - aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() - } + custom_busmap = False # snakemake.params.custom_busmap custom busmap is depreciated https://github.com/pypsa-meets-earth/pypsa-earth/pull/694 if custom_busmap: busmap = pd.read_csv( diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 92c3dd340..502cf1b9d 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -96,13 +96,12 @@ from _helpers import ( configure_logging, create_logger, - get_aggregation_strategies, + update_config_dictionary, update_p_nom_max, ) from add_electricity import load_costs from cluster_network import cluster_regions, clustering_for_n_clusters from pypsa.clustering.spatial import ( - aggregategenerators, aggregateoneport, busmap_by_stubs, get_clustering_from_busmap, @@ -276,11 +275,15 @@ def replace_components(n, c, df, pnl): _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) - _, generator_strategies = get_aggregation_strategies(aggregation_strategies) + generator_strategies = aggregation_strategies["generators"] carriers = set(n.generators.carrier) - set(exclude_carriers) - generators, generators_pnl = aggregategenerators( - n, busmap, carriers=carriers, custom_strategies=generator_strategies + generators, generators_pnl = aggregateoneport( + n, + busmap, + "Generator", + carriers=carriers, + custom_strategies=generator_strategies, ) replace_components(n, "Generator", generators, generators_pnl) @@ -588,19 +591,22 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): if not dist.empty: busmap.loc[buses_i] = dist.idxmin(1) - bus_strategies, generator_strategies = get_aggregation_strategies( - aggregation_strategies - ) + line_strategies = aggregation_strategies.get("lines", dict()) + bus_strategies = aggregation_strategies.get("buses", dict()) + generator_strategies = aggregation_strategies.get("generators", dict()) + one_port_strategies = aggregation_strategies.get("one_ports", dict()) clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, + line_strategies=line_strategies, + bus_strategies=bus_strategies, generator_strategies=generator_strategies, + one_port_strategies=one_port_strategies, scale_link_capital_costs=False, ) return clustering.network, busmap @@ -848,19 +854,22 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()): if (busmap.index == busmap).all(): return n, n.buses.index.to_series() - bus_strategies, generator_strategies = get_aggregation_strategies( - aggregation_strategies - ) + line_strategies = aggregation_strategies.get("lines", dict()) + bus_strategies = aggregation_strategies.get("buses", dict()) + generator_strategies = aggregation_strategies.get("generators", dict()) + one_port_strategies = aggregation_strategies.get("one_ports", dict()) clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, + line_strategies=line_strategies, + bus_strategies=bus_strategies, generator_strategies=generator_strategies, + one_port_strategies=one_port_strategies, scale_link_capital_costs=False, ) @@ -934,19 +943,22 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()): if (busmap.index == busmap).all(): return n, n.buses.index.to_series() - bus_strategies, generator_strategies = get_aggregation_strategies( - aggregation_strategies - ) + line_strategies = aggregation_strategies.get("lines", dict()) + bus_strategies = aggregation_strategies.get("buses", dict()) + generator_strategies = aggregation_strategies.get("generators", dict()) + one_port_strategies = aggregation_strategies.get("one_ports", dict()) clustering = get_clustering_from_busmap( n, busmap, - bus_strategies=bus_strategies, aggregate_generators_weighted=True, aggregate_generators_carriers=None, aggregate_one_ports=["Load", "StorageUnit"], line_length_factor=1.0, + line_strategies=line_strategies, + bus_strategies=bus_strategies, generator_strategies=generator_strategies, + one_port_strategies=one_port_strategies, scale_link_capital_costs=False, ) @@ -976,14 +988,27 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()): "exclude_carriers", [] ) hvdc_as_lines = snakemake.params.electricity["hvdc_as_lines"] - aggregation_strategies = snakemake.params.cluster_options.get( - "aggregation_strategies", {} + aggregation_strategies = snakemake.params.aggregation_strategies + + # Aggregation strategies must be set for all columns + update_config_dictionary( + config_dict=aggregation_strategies, + parameter_key_to_fill="lines", + dict_to_use={"v_nom": "first", "geometry": "first", "bounds": "first"}, ) - # translate str entries of aggregation_strategies to pd.Series functions: - aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() - } + update_config_dictionary( + config_dict=aggregation_strategies, + parameter_key_to_fill="buses", + dict_to_use={ + "v_nom": "first", + "lat": "mean", + "lon": "mean", + "tag_substation": "first", + "tag_area": "first", + "country": "first", + }, + ) + n, trafo_map = simplify_network_to_base_voltage(n, linetype, base_voltage) Nyears = n.snapshot_weightings.objective.sum() / 8760 @@ -1088,7 +1113,7 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()): solver_name, cluster_config.get("algorithm", "hac"), cluster_config.get("feature", None), - aggregation_strategies, + aggregation_strategies=aggregation_strategies, ) busmaps.append(cluster_map) From 994804a4b171eda0479813299a19b8d6cb4f2c4d Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Tue, 5 Nov 2024 21:34:24 +0100 Subject: [PATCH 13/68] Enable linopy - power model (#1169) * Add a zenodo link to natura.tiff * Update environment * Revise structure definition for lines * Remove get_aggregation_strategies * Fix typo aggregation_strategies * Replace aggregategenerators with aggregateoneport * Add aggregation strategies as a parameter * Re-define aggregation strategies * Update aggregation strategies * Update aggregation strategies for lines * Update aggregation strategies for buses * Fix typo * Put aggregation strategies into a variable * Parametrize the aggregation strategies * Refactor update of the aggregation strategies * Clean-up the code * Revert "Add a zenodo link to natura.tiff" This reverts commit 77007598b436df510ec8ce6f29efa9d067341628. * Define an explicit clustering strategy for v_nom * Add a release note * Get glpk back * Specify v_nom for buses explicitly * Revert "Specify v_nom for buses explicitly" This reverts commit 20192e6b3e80a2fedbee398f8e892f776bb5b5cc. * Add a version restriction to the environment specification * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Adjust naming * Move the variable definition * Move the variable * Upgrade PyPSA version * Update docstring * Fix imports duplication * Update imports * Update the carrier-capacity constraint * Add docstring * Update the equity constraint * Add docstring * Update BAU constraint * Update SAFE constraint * Add docstring * Update operational reserve margin constraint * Add docstring * Add an new argument to the RM constraint * Update the update of capacity constraints * Update adding an operational reserve margin constraint * Update docstring * Update battery constraint * Add docstring * Update a constraint related to a RES share * Fix usage of add_ERS_constraints * Update solving script * Update a solving run * Fix typos --------- Co-authored-by: Davide Fioriti Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- scripts/solve_network.py | 529 ++++++++++++++++++++++----------------- 1 file changed, 302 insertions(+), 227 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a9bbfbaa1..f52d2508b 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -52,15 +52,15 @@ linear optimal power flow (plus investment planning) is provided in the `documentation of PyPSA `_. -The optimization is based on the ``pyomo=False`` setting in the :func:`network.lopf` and :func:`pypsa.linopf.ilopf` function. -Additionally, some extra constraints specified in :mod:`prepare_network` are added. +The optimization is based on the :func:`network.optimize` function. +Additionally, some extra constraints specified in :mod:`prepare_network` and :mod:`solve_network` are added. Solving the network in multiple iterations is motivated through the dependence of transmission line capacities and impedances on values of corresponding flows. As lines are expanded their electrical parameters change, which renders the optimisation bilinear even if the power flow equations are linearized. To retain the computational advantage of continuous linear programming, a sequential linear programming technique is used, where in between iterations the line impedances are updated. -Details (and errors made through this heuristic) are discussed in the paper +Details (and errors introduced through this heuristic) are discussed in the paper - Fabian Neumann and Tom Brown. `Heuristics for Transmission Expansion Planning in Low-Carbon Energy System Models `_), *16th International Conference on the European Energy Market*, 2019. `arXiv:1907.10548 `_. @@ -86,23 +86,17 @@ import pandas as pd import pypsa from _helpers import configure_logging, create_logger, override_component_attrs +from linopy import merge from pypsa.descriptors import get_switchable_as_dense as get_as_dense -from pypsa.linopf import ( - define_constraints, - define_variables, - get_var, - ilopf, - join_exprs, - linexpr, - network_lopf, -) -from pypsa.linopt import define_constraints, get_var, join_exprs, linexpr +from pypsa.optimization.abstract import optimize_transmission_expansion_iteratively +from pypsa.optimization.optimize import optimize +from vresutils.benchmark import memory_logger logger = create_logger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) -def prepare_network(n, solve_opts): +def prepare_network(n, solve_opts, config): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -159,6 +153,25 @@ def prepare_network(n, solve_opts): def add_CCL_constraints(n, config): + """ + Add CCL (country & carrier limit) constraint to the network. + + Add minimum and maximum levels of generator nominal capacity per carrier + for individual countries. Opts and path for agg_p_nom_minmax.csv must be defined + in config.yaml. Default file is available at data/agg_p_nom_minmax.csv. + + Parameters + ---------- + n : pypsa.Network + config : dict + + Example + ------- + scenario: + opts: [Co2L-CCL-24H] + electricity: + agg_p_nom_limits: data/agg_p_nom_minmax.csv + """ agg_p_nom_limits = config["electricity"].get("agg_p_nom_limits") try: @@ -174,32 +187,57 @@ def add_CCL_constraints(n, config): ) gen_country = n.generators.bus.map(n.buses.country) - # cc means country and carrier - p_nom_per_cc = ( - pd.DataFrame( - { - "p_nom": linexpr((1, get_var(n, "Generator", "p_nom"))), - "country": gen_country, - "carrier": n.generators.carrier, - } + capacity_variable = n.model["Generator-p_nom"] + + lhs = [] + ext_carriers = n.generators.query("p_nom_extendable").carrier.unique() + for c in ext_carriers: + ext_carrier = n.generators.query("p_nom_extendable and carrier == @c") + country_grouper = ( + ext_carrier.bus.map(n.buses.country) + .rename_axis("Generator-ext") + .rename("country") ) - .dropna(subset=["p_nom"]) - .groupby(["country", "carrier"]) - .p_nom.apply(join_exprs) + ext_carrier_per_country = capacity_variable.loc[ + country_grouper.index + ].groupby_sum(country_grouper) + lhs.append(ext_carrier_per_country) + lhs = merge(lhs, dim=pd.Index(ext_carriers, name="carrier")) + + min_matrix = agg_p_nom_minmax["min"].to_xarray().unstack().reindex_like(lhs) + max_matrix = agg_p_nom_minmax["max"].to_xarray().unstack().reindex_like(lhs) + + n.model.add_constraints( + lhs >= min_matrix, name="agg_p_nom_min", mask=min_matrix.notnull() + ) + n.model.add_constraints( + lhs <= max_matrix, name="agg_p_nom_max", mask=max_matrix.notnull() ) - minimum = agg_p_nom_minmax["min"].dropna() - if not minimum.empty: - minconstraint = define_constraints( - n, p_nom_per_cc[minimum.index], ">=", minimum, "agg_p_nom", "min" - ) - maximum = agg_p_nom_minmax["max"].dropna() - if not maximum.empty: - maxconstraint = define_constraints( - n, p_nom_per_cc[maximum.index], "<=", maximum, "agg_p_nom", "max" - ) def add_EQ_constraints(n, o, scaling=1e-1): + """ + Add equity constraints to the network. + + Currently this is only implemented for the electricity sector only. + + Opts must be specified in the config.yaml. + + Parameters + ---------- + n : pypsa.Network + o : str + + Example + ------- + scenario: + opts: [Co2L-EQ0.7-24h] + + Require each country or node to on average produce a minimal share + of its total electricity consumption itself. Example: EQ0.7c demands each country + to produce on average at least 70% of its consumption; EQ0.7 demands + each node to produce on average at least 70% of its consumption. + """ float_regex = "[0-9]*\.?[0-9]+" level = float(re.findall(float_regex, o)[0]) if o[-1] == "c": @@ -220,99 +258,150 @@ def add_EQ_constraints(n, o, scaling=1e-1): ) inflow = inflow.reindex(load.index).fillna(0.0) rhs = scaling * (level * load - inflow) + dispatch_variable = n.model["Generator-p"] lhs_gen = ( - linexpr( - (n.snapshot_weightings.generators * scaling, get_var(n, "Generator", "p").T) - ) - .T.groupby(ggrouper, axis=1) - .apply(join_exprs) + (dispatch_variable * (n.snapshot_weightings.generators * scaling)) + .groupby(ggrouper.to_xarray()) + .sum() + .sum("snapshot") ) - lhs_spill = ( - linexpr( - ( - -n.snapshot_weightings.stores * scaling, - get_var(n, "StorageUnit", "spill").T, - ) + # TODO: double check that this is really needed, why do have to subtract the spillage + if not n.storage_units_t.inflow.empty: + spillage_variable = n.model["StorageUnit-spill"] + lhs_spill = ( + (spillage_variable * (-n.snapshot_weightings.stores * scaling)) + .groupby_sum(sgrouper) + .groupby(sgrouper.to_xarray()) + .sum() + .sum("snapshot") ) - .T.groupby(sgrouper, axis=1) - .apply(join_exprs) - ) - lhs_spill = lhs_spill.reindex(lhs_gen.index).fillna("") - lhs = lhs_gen + lhs_spill - define_constraints(n, lhs, ">=", rhs, "equity", "min") + lhs = lhs_gen + lhs_spill + else: + lhs = lhs_gen + n.model.add_constraints(lhs >= rhs, name="equity_min") def add_BAU_constraints(n, config): - ext_c = n.generators.query("p_nom_extendable").carrier.unique() - mincaps = pd.Series( - config["electricity"].get("BAU_mincapacities", {key: 0 for key in ext_c}) - ) - lhs = ( - linexpr((1, get_var(n, "Generator", "p_nom"))) - .groupby(n.generators.carrier) - .apply(join_exprs) - ) - define_constraints(n, lhs, ">=", mincaps[lhs.index], "Carrier", "bau_mincaps") - - maxcaps = pd.Series( - config["electricity"].get("BAU_maxcapacities", {key: np.inf for key in ext_c}) - ) - lhs = ( - linexpr((1, get_var(n, "Generator", "p_nom"))) - .groupby(n.generators.carrier) - .apply(join_exprs) - ) - define_constraints(n, lhs, "<=", maxcaps[lhs.index], "Carrier", "bau_maxcaps") + """ + Add a per-carrier minimal overall capacity. + + BAU_mincapacities and opts must be adjusted in the config.yaml. + + Parameters + ---------- + n : pypsa.Network + config : dict + + Example + ------- + scenario: + opts: [Co2L-BAU-24h] + electricity: + BAU_mincapacities: + solar: 0 + onwind: 0 + OCGT: 100000 + offwind-ac: 0 + offwind-dc: 0 + Which sets minimum expansion across all nodes e.g. in Europe to 100GW. + OCGT bus 1 + OCGT bus 2 + ... > 100000 + """ + mincaps = pd.Series(config["electricity"]["BAU_mincapacities"]) + p_nom = n.model["Generator-p_nom"] + ext_i = n.generators.query("p_nom_extendable") + ext_carrier_i = xr.DataArray(ext_i.carrier.rename_axis("Generator-ext")) + lhs = p_nom.groupby(ext_carrier_i).sum() + rhs = mincaps[lhs.indexes["carrier"]].rename_axis("carrier") + n.model.add_constraints(lhs >= rhs, name="bau_mincaps") def add_SAFE_constraints(n, config): - peakdemand = ( - 1.0 + config["electricity"]["SAFE_reservemargin"] - ) * n.loads_t.p_set.sum(axis=1).max() - conv_techs = config["plotting"]["conv_techs"] + """ + Add a capacity reserve margin of a certain fraction above the peak demand. + Renewable generators and storage do not contribute. Ignores network. + + Parameters + ---------- + n : pypsa.Network + config : dict + + Example + ------- + config.yaml requires to specify opts: + + scenario: + opts: [Co2L-SAFE-24h] + electricity: + SAFE_reservemargin: 0.1 + Which sets a reserve margin of 10% above the peak demand. + """ + peakdemand = n.loads_t.p_set.sum(axis=1).max() + margin = 1.0 + config["electricity"]["SAFE_reservemargin"] + reserve_margin = peakdemand * margin + conventional_carriers = config["electricity"]["conventional_carriers"] + ext_gens_i = n.generators.query( + "carrier in @conventional_carriers & p_nom_extendable" + ).index + capacity_variable = n.model["Generator-p_nom"] + p_nom = n.model["Generator-p_nom"].loc[ext_gens_i] + lhs = p_nom.sum() exist_conv_caps = n.generators.query( - "~p_nom_extendable & carrier in @conv_techs" + "~p_nom_extendable & carrier in @entional_carriers" ).p_nom.sum() - ext_gens_i = n.generators.query("carrier in @conv_techs & p_nom_extendable").index - lhs = linexpr((1, get_var(n, "Generator", "p_nom")[ext_gens_i])).sum() - rhs = peakdemand - exist_conv_caps - define_constraints(n, lhs, ">=", rhs, "Safe", "mintotalcap") + rhs = reserve_margin - exist_conv_caps + n.model.add_constraints(lhs >= rhs, name="safe_mintotalcap") -def add_operational_reserve_margin_constraint(n, config): +def add_operational_reserve_margin_constraint(n, sns, config): + """ + Build reserve margin constraints based on the formulation + as suggested in GenX + https://energy.mit.edu/wp-content/uploads/2017/10/Enhanced-Decision-Support-for-a-Changing-Electricity-Landscape.pdf + It implies that the reserve margin also accounts for optimal + dispatch of distributed energy resources (DERs) and demand response + which is a novel feature of GenX. + """ reserve_config = config["electricity"]["operational_reserve"] EPSILON_LOAD = reserve_config["epsilon_load"] EPSILON_VRES = reserve_config["epsilon_vres"] CONTINGENCY = reserve_config["contingency"] # Reserve Variables - reserve = get_var(n, "Generator", "r") - lhs = linexpr((1, reserve)).sum(1) + n.model.add_variables( + 0, np.inf, coords=[sns, n.generators.index], name="Generator-r" + ) + reserve = n.model["Generator-r"] + lhs = reserve.sum("Generator") # Share of extendable renewable capacities ext_i = n.generators.query("p_nom_extendable").index vres_i = n.generators_t.p_max_pu.columns if not ext_i.empty and not vres_i.empty: capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] - renewable_capacity_variables = get_var(n, "Generator", "p_nom")[ - vres_i.intersection(ext_i) - ] - lhs += linexpr( - (-EPSILON_VRES * capacity_factor, renewable_capacity_variables) - ).sum(1) + renewable_capacity_variables = ( + n.model["Generator-p_nom"] + .loc[vres_i.intersection(ext_i)] + .rename({"Generator-ext": "Generator"}) + ) + lhs = merge( + lhs, + (renewable_capacity_variables * (-EPSILON_VRES * capacity_factor)).sum( + ["Generator"] + ), + ) - # Total demand at t - demand = n.loads_t.p.sum(1) + # Total demand per t + demand = get_as_dense(n, "Load", "p_set").sum(axis=1) # VRES potential of non extendable generators capacity_factor = n.generators_t.p_max_pu[vres_i.difference(ext_i)] renewable_capacity = n.generators.p_nom[vres_i.difference(ext_i)] - potential = (capacity_factor * renewable_capacity).sum(1) + potential = (capacity_factor * renewable_capacity).sum(axis=1) # Right-hand-side rhs = EPSILON_LOAD * demand + EPSILON_VRES * potential + CONTINGENCY - define_constraints(n, lhs, ">=", rhs, "Reserve margin") + n.model.add_constraints(lhs >= rhs, name="reserve_margin") def update_capacity_constraint(n): @@ -320,65 +409,84 @@ def update_capacity_constraint(n): ext_i = n.generators.query("p_nom_extendable").index fix_i = n.generators.query("not p_nom_extendable").index - dispatch = get_var(n, "Generator", "p") - reserve = get_var(n, "Generator", "r") + dispatch = n.model["Generator-p"] + reserve = n.model["Generator-r"] capacity_fixed = n.generators.p_nom[fix_i] p_max_pu = get_as_dense(n, "Generator", "p_max_pu") - lhs = linexpr((1, dispatch), (1, reserve)) + lhs = merge( + dispatch * 1, + reserve * 1, + ) if not ext_i.empty: - capacity_variable = get_var(n, "Generator", "p_nom") - lhs += linexpr((-p_max_pu[ext_i], capacity_variable)).reindex( - columns=gen_i, fill_value="" - ) + capacity_variable = n.model["Generator-p_nom"] + lhs = dispatch + reserve - capacity_variable * xr.DataArray(p_max_pu[ext_i]) rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0) - define_constraints(n, lhs, "<=", rhs, "Generators", "updated_capacity_constraint") + n.model.add_constraints( + lhs <= rhs, name="gen_updated_capacity_constraint", mask=rhs.notnull() + ) def add_operational_reserve_margin(n, sns, config): """ - Build reserve margin constraints based on the formulation given in - https://genxproject.github.io/GenX/dev/core/#Reserves. + Parameters + ---------- + n : pypsa.Network + sns: pd.DatetimeIndex + config : dict + + Example: + -------- + config.yaml requires to specify operational_reserve: + operational_reserve: # like https://genxproject.github.io/GenX/dev/core/#Reserves + activate: true + epsilon_load: 0.02 # percentage of load at each snapshot + epsilon_vres: 0.02 # percentage of VRES at each snapshot + contingency: 400000 # MW """ - define_variables(n, 0, np.inf, "Generator", "r", axes=[sns, n.generators.index]) - - add_operational_reserve_margin_constraint(n, config) + add_operational_reserve_margin_constraint(n, sns, config) update_capacity_constraint(n) def add_battery_constraints(n): + """ + Add constraint ensuring that charger = discharger, i.e. + 1 * charger_size - efficiency * discharger_size = 0 + """ nodes = n.buses.index[n.buses.carrier == "battery"] - if nodes.empty or ("Link", "p_nom") not in n.variables.index: + # TODO Check if the second part of the condition can make sense + # if nodes.empty or ("Link", "p_nom") not in n.variables.index: + if nodes.empty: return - link_p_nom = get_var(n, "Link", "p_nom") - lhs = linexpr( - (1, link_p_nom[nodes + " charger"]), - ( - -n.links.loc[nodes + " discharger", "efficiency"].values, - link_p_nom[nodes + " discharger"].values, - ), + vars_link = n.model["Link-p_nom"] + eff = n.links.loc[nodes + " discharger", "efficiency"] + lhs = merge( + vars_link.sel({"Link-ext": nodes + " charger"}) * 1, + # for some reasons, eff is one element longer as compared with vars_link + vars_link.sel({"Link-ext": nodes + " discharger"}) * -eff[0], ) - define_constraints(n, lhs, "=", 0, "Link", "charger_ratio") + n.model.add_constraints(lhs == 0, name="link_charger_ratio") -def add_RES_constraints(n, res_share): +def add_RES_constraints(n, res_share, config): lgrouper = n.loads.bus.map(n.buses.country) + # TODO drop load ggrouper = n.generators.bus.map(n.buses.country) sgrouper = n.storage_units.bus.map(n.buses.country) cgrouper = n.links.bus0.map(n.buses.country) logger.warning( - "The add_RES_constraints functionality is still work in progress. " + "The add_RES_constraints() is still work in progress. " "Unexpected results might be incurred, particularly if " "temporal clustering is applied or if an unexpected change of technologies " - "is subject to the obtimisation." + "is subject to future improvements." ) load = ( @@ -388,103 +496,68 @@ def add_RES_constraints(n, res_share): rhs = res_share * load - res_techs = [ - "solar", - "onwind", - "offwind-dc", - "offwind-ac", - "battery", - "hydro", - "ror", - ] + renew_techs = config["electricity"]["renewable_carriers"] + charger = ["H2 electrolysis", "battery charger"] discharger = ["H2 fuel cell", "battery discharger"] - gens_i = n.generators.query("carrier in @res_techs").index - stores_i = n.storage_units.query("carrier in @res_techs").index + gens_i = n.generators.query("carrier in @renew_techs").index + stores_i = n.storage_units.query("carrier in @renew_techs").index + charger_i = n.links.query("carrier in @charger").index discharger_i = n.links.query("carrier in @discharger").index + stores_t_weights = n.snapshot_weightings.stores + # Generators + # TODO restore grouping by countries un-commenting calls of groupby() lhs_gen = ( - linexpr( - (n.snapshot_weightings.generators, get_var(n, "Generator", "p")[gens_i].T) - ) - .T.groupby(ggrouper, axis=1) - .apply(join_exprs) + (n.model["Generator-p"].loc[:, gens_i] * n.snapshot_weightings.generators) + # .groupby(ggrouper.to_xarray()) + .sum() ) # StorageUnits - lhs_dispatch = ( - ( - linexpr( - ( - n.snapshot_weightings.stores, - get_var(n, "StorageUnit", "p_dispatch")[stores_i].T, - ) - ) - .T.groupby(sgrouper, axis=1) - .apply(join_exprs) - ) - .reindex(lhs_gen.index) - .fillna("") + store_disp_expr = ( + n.model["StorageUnit-p_dispatch"].loc[:, stores_i] * stores_t_weights + ) + store_expr = n.model["StorageUnit-p_store"].loc[:, stores_i] * stores_t_weights + charge_expr = n.model["Link-p"].loc[:, charger_i] * stores_t_weights.apply( + lambda r: r * n.links.loc[charger_i].efficiency + ) + discharge_expr = n.model["Link-p"].loc[:, discharger_i] * stores_t_weights.apply( + lambda r: r * n.links.loc[discharger_i].efficiency ) + lhs_dispatch = ( + store_disp_expr + # .groupby(sgrouper) + .sum() + ) lhs_store = ( - ( - linexpr( - ( - -n.snapshot_weightings.stores, - get_var(n, "StorageUnit", "p_store")[stores_i].T, - ) - ) - .T.groupby(sgrouper, axis=1) - .apply(join_exprs) - ) - .reindex(lhs_gen.index) - .fillna("") + store_expr + # .groupby(sgrouper) + .sum() ) # Stores (or their resp. Link components) # Note that the variables "p0" and "p1" currently do not exist. # Thus, p0 and p1 must be derived from "p" (which exists), taking into account the link efficiency. lhs_charge = ( - ( - linexpr( - ( - -n.snapshot_weightings.stores, - get_var(n, "Link", "p")[charger_i].T, - ) - ) - .T.groupby(cgrouper, axis=1) - .apply(join_exprs) - ) - .reindex(lhs_gen.index) - .fillna("") + charge_expr + # .groupby(cgrouper) + .sum() ) lhs_discharge = ( - ( - linexpr( - ( - n.snapshot_weightings.stores.apply( - lambda r: r * n.links.loc[discharger_i].efficiency - ), - get_var(n, "Link", "p")[discharger_i], - ) - ) - .groupby(cgrouper, axis=1) - .apply(join_exprs) - ) - .reindex(lhs_gen.index) - .fillna("") + discharge_expr + # .groupby(cgrouper) + .sum() ) - # signs of resp. terms are coded in the linexpr. - # todo: for links (lhs_charge and lhs_discharge), account for snapshot weightings - lhs = lhs_gen + lhs_dispatch + lhs_store + lhs_charge + lhs_discharge + lhs = lhs_gen + lhs_dispatch - lhs_store - lhs_charge + lhs_discharge - define_constraints(n, lhs, "=", rhs, "RES share") + n.model.add_constraints(lhs == rhs, name="res_share") def add_land_use_constraint(n): @@ -876,7 +949,7 @@ def extra_functionality(n, snapshots): for o in opts: if "RES" in o: res_share = float(re.findall("[0-9]*\.?[0-9]+$", o)[0]) - add_RES_constraints(n, res_share) + add_RES_constraints(n, res_share, config) for o in opts: if "EQ" in o: add_EQ_constraints(n, o) @@ -927,40 +1000,44 @@ def extra_functionality(n, snapshots): add_co2_sequestration_limit(n, snapshots) -def solve_network(n, config, solving={}, opts="", **kwargs): +def solve_network(n, config, solving, **kwargs): set_of_options = solving["solver"]["options"] cf_solving = solving["options"] - solver_options = solving["solver_options"][set_of_options] if set_of_options else {} - solver_name = solving["solver"]["name"] + kwargs["solver_options"] = ( + solving["solver_options"][set_of_options] if set_of_options else {} + ) + kwargs["solver_name"] = solving["solver"]["name"] - track_iterations = cf_solving.get("track_iterations", False) - min_iterations = cf_solving.get("min_iterations", 4) - max_iterations = cf_solving.get("max_iterations", 6) + skip_iterations = cf_solving.get("skip_iterations", False) + if not n.lines.s_nom_extendable.any(): + skip_iterations = True + logger.info("No expandable lines found. Skipping iterative solving.") # add to network for extra_functionality n.config = config n.opts = opts - if cf_solving.get("skip_iterations", False): - network_lopf( - n, - solver_name=solver_name, - solver_options=solver_options, - extra_functionality=extra_functionality, - **kwargs, - ) + if skip_iterations: + status, condition = n.optimize(**kwargs) else: - ilopf( - n, - solver_name=solver_name, - solver_options=solver_options, - track_iterations=track_iterations, - min_iterations=min_iterations, - max_iterations=max_iterations, - extra_functionality=extra_functionality, - **kwargs, + kwargs["track_iterations"] = (cf_solving.get("track_iterations", False),) + kwargs["min_iterations"] = (cf_solving.get("min_iterations", 4),) + kwargs["max_iterations"] = (cf_solving.get("max_iterations", 6),) + status, condition = n.optimize.optimize_transmission_expansion_iteratively( + **kwargs ) + + if status != "ok": # and not rolling_horizon: + logger.warning( + f"Solving status '{status}' with termination condition '{condition}'" + ) + if "infeasible" in condition: + labels = n.model.compute_infeasibilities() + logger.info(f"Labels:\n{labels}") + n.model.print_infeasibilities() + raise RuntimeError("Solving status 'infeasible'") + return n @@ -978,11 +1055,8 @@ def solve_network(n, config, solving={}, opts="", **kwargs): configure_logging(snakemake) - tmpdir = snakemake.params.solving.get("tmpdir") - if tmpdir is not None: - Path(tmpdir).mkdir(parents=True, exist_ok=True) opts = snakemake.wildcards.opts.split("-") - solving = snakemake.params.solving + solve_opts = snakemake.config["solving"]["options"] is_sector_coupled = "sopts" in snakemake.wildcards.keys() @@ -992,10 +1066,11 @@ def solve_network(n, config, solving={}, opts="", **kwargs): else: n = pypsa.Network(snakemake.input.network) - if snakemake.params.augmented_line_connection.get("add_to_snakefile"): - n.lines.loc[n.lines.index.str.contains("new"), "s_nom_min"] = ( - snakemake.params.augmented_line_connection.get("min_expansion") - ) + # TODO Double-check handling the augmented case + # if snakemake.params.augmented_line_connection.get("add_to_snakefile"): + # n.lines.loc[n.lines.index.str.contains("new"), "s_nom_min"] = ( + # snakemake.params.augmented_line_connection.get("min_expansion") + # ) if ( snakemake.config["custom_data"]["add_existing"] @@ -1016,15 +1091,15 @@ def solve_network(n, config, solving={}, opts="", **kwargs): else: n_ref = None - n = prepare_network(n, solving["options"]) + # needed to get `n.model` property + n.optimize.create_model() + n = prepare_network(n, solve_opts, config=solve_opts) n = solve_network( n, config=snakemake.config, - solving=solving, - opts=opts, - solver_dir=tmpdir, - solver_logfile=snakemake.log.solver, + solving=snakemake.params.solving, + log_fn=snakemake.log.solver, ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) n.export_to_netcdf(snakemake.output[0]) From 0d3ebdf454b8893c623d96e20fda4a257667d9a4 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 5 Nov 2024 21:42:59 +0100 Subject: [PATCH 14/68] Get back CI for Windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5887ac014..08f69a702 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: os: - ubuntu-latest - macos-latest - # - windows-latest + - windows-latest runs-on: ${{ matrix.os }} From 34b616894ca693e1c822d385969ed651f55e80ce Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 5 Nov 2024 21:43:38 +0100 Subject: [PATCH 15/68] Update environment --- envs/environment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index dc0726ebe..e263e7a78 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -78,7 +78,6 @@ dependencies: # Default solver for tests (required for CI) - glpk -- ipopt - gurobi - pip: From 268851b280933ab0c3b3adce4334de75cd89d14c Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 7 Nov 2024 17:16:15 +0100 Subject: [PATCH 16/68] revise implementation to accomodate linopy for sec --- scripts/solve_network.py | 148 ++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index f52d2508b..0bd087016 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -621,16 +621,21 @@ def _add_land_use_constraint_m(n): def add_h2_network_cap(n, cap): h2_network = n.links.loc[n.links.carrier == "H2 pipeline"] - if h2_network.index.empty or ("Link", "p_nom") not in n.variables.index: + if h2_network.index.empty: return - h2_network_cap = get_var(n, "Link", "p_nom") - subset_index = h2_network.index.intersection(h2_network_cap.index) - lhs = linexpr( - (h2_network.loc[subset_index, "length"], h2_network_cap[subset_index]) + h2_network_cap = n.model["Link-p_nom"] + h2_network_cap_index = h2_network_cap.indexes["Link-ext"] + subset_index = h2_network.index.intersection(h2_network_cap_index) + diff_index = h2_network_cap_index.difference(subset_index) + if len(diff_index) > 0: + logger.warning( + f"Impossible to set H2 cap for the following links: {diff_index}" + ) + lhs = ( + h2_network_cap.loc[subset_index] * h2_network.loc[subset_index, "length"] ).sum() - # lhs = linexpr((1, h2_network_cap[h2_network.index])).sum() rhs = cap * 1000 - define_constraints(n, lhs, "<=", rhs, "h2_network_cap") + n.model.add_constraints(lhs <= rhs, name="h2_network_cap") def H2_export_yearly_constraint(n): @@ -651,9 +656,10 @@ def H2_export_yearly_constraint(n): index=n.snapshots, columns=res_index, ) - res = join_exprs( - linexpr((weightings, get_var(n, "Generator", "p")[res_index])) - ) # single line sum + capacity_variable = n.model["Generator-p"] + + # single line sum + res = (weightings * capacity_variable.loc[res_index]).sum() load_ind = n.loads[n.loads.carrier == "AC"].index.intersection( n.loads_t.p_set.columns @@ -681,7 +687,7 @@ def H2_export_yearly_constraint(n): else: rhs = h2_export * (1 / 0.7) - con = define_constraints(n, lhs, ">=", rhs, "H2ExportConstraint", "RESproduction") + n.model.add_constraints(lhs >= rhs, name="H2ExportConstraint-RESproduction") def monthly_constraints(n, n_ref): @@ -704,15 +710,17 @@ def monthly_constraints(n, n_ref): index=n.snapshots, columns=res_index, ) + capacity_variable = n.model["Generator-p"] - res = linexpr((weightings, get_var(n, "Generator", "p")[res_index])).sum( - axis=1 - ) # single line sum + # single line sum + res = (weightings * capacity_variable[res_index]).sum(axis=1) res = res.groupby(res.index.month).sum() - electrolysis = get_var(n, "Link", "p")[ + link_p = n.model["Link-p"] + electrolysis = link_p.loc[ n.links.index[n.links.index.str.contains("H2 Electrolysis")] ] + weightings_electrolysis = pd.DataFrame( np.outer( n.snapshot_weightings["generators"], [1.0] * len(electrolysis.columns) @@ -721,7 +729,7 @@ def monthly_constraints(n, n_ref): columns=electrolysis.columns, ) - elec_input = linexpr((-allowed_excess * weightings_electrolysis, electrolysis)).sum( + elec_input = ((-allowed_excess * weightings_electrolysis) * electrolysis).sum( axis=1 ) @@ -744,16 +752,16 @@ def monthly_constraints(n, n_ref): for i in range(len(res.index)): lhs = res.iloc[i] + "\n" + elec_input.iloc[i] rhs = res_ref.iloc[i] + elec_input_ref.iloc[i] - con = define_constraints( - n, lhs, ">=", rhs, f"RESconstraints_{i}", f"REStarget_{i}" + n.model.add_constraints( + lhs >= rhs, name=f"RESconstraints_{i}-REStarget_{i}" ) else: for i in range(len(res.index)): lhs = res.iloc[i] + "\n" + elec_input.iloc[i] - con = define_constraints( - n, lhs, ">=", 0.0, f"RESconstraints_{i}", f"REStarget_{i}" + n.model.add_constraints( + lhs >= 0.0, name=f"RESconstraints_{i}-REStarget_{i}" ) # else: # logger.info("ignoring H2 export constraint as wildcard is set to 0") @@ -774,84 +782,72 @@ def add_chp_constraints(n): electric = n.links.index[electric_bool] heat = n.links.index[heat_bool] - electric_ext = n.links.index[electric_bool & n.links.p_nom_extendable] - heat_ext = n.links.index[heat_bool & n.links.p_nom_extendable] + electric_ext = n.links[electric_bool].query("p_nom_extendable").index + heat_ext = n.links[heat_bool].query("p_nom_extendable").index - electric_fix = n.links.index[electric_bool & ~n.links.p_nom_extendable] - heat_fix = n.links.index[heat_bool & ~n.links.p_nom_extendable] + electric_fix = n.links[electric_bool].query("~p_nom_extendable").index + heat_fix = n.links[heat_bool].query("~p_nom_extendable").index - link_p = get_var(n, "Link", "p") + p = n.model["Link-p"] # dimension: [time, link] + # output ratio between heat and electricity and top_iso_fuel_line for extendable if not electric_ext.empty: - link_p_nom = get_var(n, "Link", "p_nom") - - # ratio of output heat to electricity set by p_nom_ratio - lhs = linexpr( - ( - n.links.loc[electric_ext, "efficiency"] - * n.links.loc[electric_ext, "p_nom_ratio"], - link_p_nom[electric_ext], - ), - (-n.links.loc[heat_ext, "efficiency"].values, link_p_nom[heat_ext].values), - ) + p_nom = n.model["Link-p_nom"] - define_constraints(n, lhs, "=", 0, "chplink", "fix_p_nom_ratio") - - # top_iso_fuel_line for extendable - lhs = linexpr( - (1, link_p[heat_ext]), - (1, link_p[electric_ext].values), - (-1, link_p_nom[electric_ext].values), + lhs = ( + p_nom.loc[electric_ext] + * (n.links.p_nom_ratio * n.links.efficiency)[electric_ext].values + - p_nom.loc[heat_ext] * n.links.efficiency[heat_ext].values ) + n.model.add_constraints(lhs == 0, name="chplink-fix_p_nom_ratio") - define_constraints(n, lhs, "<=", 0, "chplink", "top_iso_fuel_line_ext") + rename = {"Link-ext": "Link"} + lhs = ( + p.loc[:, electric_ext] + + p.loc[:, heat_ext] + - p_nom.rename(rename).loc[electric_ext] + ) + n.model.add_constraints(lhs <= 0, name="chplink-top_iso_fuel_line_ext") + # top_iso_fuel_line for fixed if not electric_fix.empty: - # top_iso_fuel_line for fixed - lhs = linexpr((1, link_p[heat_fix]), (1, link_p[electric_fix].values)) - - rhs = n.links.loc[electric_fix, "p_nom"].values - - define_constraints(n, lhs, "<=", rhs, "chplink", "top_iso_fuel_line_fix") + lhs = p.loc[:, electric_fix] + p.loc[:, heat_fix] + rhs = n.links.p_nom[electric_fix] + n.model.add_constraints(lhs <= rhs, name="chplink-top_iso_fuel_line_fix") + # back-pressure if not electric.empty: - # backpressure - lhs = linexpr( - ( - n.links.loc[electric, "c_b"].values * n.links.loc[heat, "efficiency"], - link_p[heat], - ), - (-n.links.loc[electric, "efficiency"].values, link_p[electric].values), + lhs = ( + p.loc[:, heat] * (n.links.efficiency[heat] * n.links.c_b[electric].values) + - p.loc[:, electric] * n.links.efficiency[electric] ) - - define_constraints(n, lhs, "<=", 0, "chplink", "backpressure") + n.model.add_constraints(lhs <= rhs, name="chplink-backpressure") def add_co2_sequestration_limit(n, sns): co2_stores = n.stores.loc[n.stores.carrier == "co2 stored"].index - if co2_stores.empty or ("Store", "e") not in n.variables.index: + if co2_stores.empty: # or ("Store", "e") not in n.variables.index: return - vars_final_co2_stored = get_var(n, "Store", "e").loc[sns[-1], co2_stores] + vars_final_co2_stored = n.model["Store-e"].loc[sns[-1], co2_stores] - lhs = linexpr((1, vars_final_co2_stored)).sum() + lhs = (1 * vars_final_co2_stored).sum() rhs = ( n.config["sector"].get("co2_sequestration_potential", 5) * 1e6 ) # TODO change 200 limit (Europe) name = "co2_sequestration_limit" - define_constraints( - n, lhs, "<=", rhs, "GlobalConstraint", "mu", axes=pd.Index([name]), spec=name - ) + + n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") def set_h2_colors(n): - blue_h2 = get_var(n, "Link", "p")[ + blue_h2 = n.model["Link-p"].loc[ n.links.index[n.links.index.str.contains("blue H2")] ] - pink_h2 = get_var(n, "Link", "p")[ + pink_h2 = n.model["Link-p"].loc[ n.links.index[n.links.index.str.contains("pink H2")] ] @@ -883,16 +879,16 @@ def set_h2_colors(n): columns=pink_h2.columns, ) - total_blue = linexpr((weightings_blue, blue_h2)).sum().sum() + total_blue = (weightings_blue * blue_h2).sum().sum() - total_pink = linexpr((weightings_pink, pink_h2)).sum().sum() + total_pink = (weightings_pink * pink_h2).sum().sum() rhs_blue = load_h2 * snakemake.config["sector"]["hydrogen"]["blue_share"] rhs_pink = load_h2 * snakemake.config["sector"]["hydrogen"]["pink_share"] - define_constraints(n, total_blue, "=", rhs_blue, "blue_h2_share") + n.model.add_constraints(total_blue == rhs_blue, name="blue_h2_share") - define_constraints(n, total_pink, "=", rhs_pink, "pink_h2_share") + n.model.add_constraints(total_pink == rhs_pink, name="pink_h2_share") def add_existing(n): @@ -953,8 +949,13 @@ def extra_functionality(n, snapshots): for o in opts: if "EQ" in o: add_EQ_constraints(n, o) + add_battery_constraints(n) + if snakemake.config["sector"]["chp"]: + logger.info("setting CHP constraints") + add_chp_constraints(n) + if ( snakemake.config["policy_config"]["hydrogen"]["temporal_matching"] == "h2_yearly_matching" @@ -1008,6 +1009,7 @@ def solve_network(n, config, solving, **kwargs): solving["solver_options"][set_of_options] if set_of_options else {} ) kwargs["solver_name"] = solving["solver"]["name"] + kwargs["extra_functionality"] = extra_functionality skip_iterations = cf_solving.get("skip_iterations", False) if not n.lines.s_nom_extendable.any(): @@ -1048,9 +1050,9 @@ def solve_network(n, config, solving, **kwargs): snakemake = mock_snakemake( "solve_network", simpl="", - clusters="54", + clusters="10", ll="copt", - opts="Co2L-1H", + opts="Co2L-3H", ) configure_logging(snakemake) From 9fcee64a2e9675a09488b102e64d7b7d5c6299c5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:51:37 +0000 Subject: [PATCH 17/68] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index cc16a016b..0bd087016 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1011,7 +1011,6 @@ def solve_network(n, config, solving, **kwargs): kwargs["solver_name"] = solving["solver"]["name"] kwargs["extra_functionality"] = extra_functionality - skip_iterations = cf_solving.get("skip_iterations", False) if not n.lines.s_nom_extendable.any(): skip_iterations = True From 08bd2bde9f01a78438e180f41c08c364372a2317 Mon Sep 17 00:00:00 2001 From: ekatef Date: Fri, 15 Nov 2024 23:37:35 +0100 Subject: [PATCH 18/68] Update PyPSA version --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index e263e7a78..6b8da543b 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -12,7 +12,7 @@ dependencies: - pip - mamba # esp for windows build -- pypsa>=0.25, <0.29 +- pypsa>=0.25, <0.31 # - atlite>=0.2.4 # until https://github.com/PyPSA/atlite/issues/244 is not merged - dask - powerplantmatching From 5264e625d848456b0275d79679cb23c2d63ca1cc Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 19 Nov 2024 23:24:11 +0100 Subject: [PATCH 19/68] Lift a version restriction --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 6b8da543b..f345839ff 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -12,7 +12,7 @@ dependencies: - pip - mamba # esp for windows build -- pypsa>=0.25, <0.31 +- pypsa>=0.25 # - atlite>=0.2.4 # until https://github.com/PyPSA/atlite/issues/244 is not merged - dask - powerplantmatching From cedfda8aa5e3455579640e675eb4ad5602f96e3f Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 19 Nov 2024 23:24:50 +0100 Subject: [PATCH 20/68] Enable augmented network --- scripts/solve_network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index f52d2508b..a446d8787 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1067,10 +1067,10 @@ def solve_network(n, config, solving, **kwargs): n = pypsa.Network(snakemake.input.network) # TODO Double-check handling the augmented case - # if snakemake.params.augmented_line_connection.get("add_to_snakefile"): - # n.lines.loc[n.lines.index.str.contains("new"), "s_nom_min"] = ( - # snakemake.params.augmented_line_connection.get("min_expansion") - # ) + if snakemake.params.augmented_line_connection.get("add_to_snakefile"): + n.lines.loc[n.lines.index.str.contains("new"), "s_nom_min"] = ( + snakemake.params.augmented_line_connection.get("min_expansion") + ) if ( snakemake.config["custom_data"]["add_existing"] From f61058aed892f272f841ea0c2f02a086145688f7 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 20 Nov 2024 10:35:17 +0100 Subject: [PATCH 21/68] Add a requirement to update countrycode --- envs/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/envs/environment.yaml b/envs/environment.yaml index f345839ff..1e2f43e20 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -88,3 +88,4 @@ dependencies: - tsam>=1.1.0 - chaospy # lastest version only available on pip - fake_useragent + - countrycode==0.4.0 # a sub-dependency; the restriction is needed to deal with an encoding error for windows From c5a47646073462ff71f85425ce51e9fb63841c4e Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 20 Nov 2024 12:09:36 +0100 Subject: [PATCH 22/68] Use an installation from the source for countrycode --- envs/environment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 1e2f43e20..b23ab6d44 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -85,7 +85,8 @@ dependencies: - powerplantmatching>=0.5.19 # until conda release it out - git+https://github.com/davide-f/google-drive-downloader@master # google drive with fix for virus scan - git+https://github.com/FRESNA/vresutils@master # until new pip release > 0.3.1 (strictly) + - git+https://github.com/vincentarelbundock/pycountrycode@main # until the fix will be available via https://github.com/vincentarelbundock/pycountrycode/pull/13 - tsam>=1.1.0 - chaospy # lastest version only available on pip - fake_useragent - - countrycode==0.4.0 # a sub-dependency; the restriction is needed to deal with an encoding error for windows + #- countrycode==0.4.0 # a sub-dependency; the restriction is needed to deal with an encoding error for windows From acc94319900448c4a42a9407b6ee6a35cf164a66 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 20 Nov 2024 21:58:42 +0100 Subject: [PATCH 23/68] Remove redundand initialisation --- scripts/solve_network.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a446d8787..324cda356 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1091,8 +1091,6 @@ def solve_network(n, config, solving, **kwargs): else: n_ref = None - # needed to get `n.model` property - n.optimize.create_model() n = prepare_network(n, solve_opts, config=solve_opts) n = solve_network( From 04065eb5c7fb931e9c27bdac2f865acdd939668f Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 20 Nov 2024 23:15:08 +0100 Subject: [PATCH 24/68] Switch on extra functionality --- scripts/solve_network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 324cda356..845acd64b 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1008,6 +1008,7 @@ def solve_network(n, config, solving, **kwargs): solving["solver_options"][set_of_options] if set_of_options else {} ) kwargs["solver_name"] = solving["solver"]["name"] + kwargs["extra_functionality"] = extra_functionality skip_iterations = cf_solving.get("skip_iterations", False) if not n.lines.s_nom_extendable.any(): From dc4dba3309ee749f8bd8a728a747512117ef5767 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 20 Nov 2024 23:29:53 +0100 Subject: [PATCH 25/68] Fix grouping in add_res_constraints --- scripts/solve_network.py | 64 ++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 845acd64b..41b5f42cb 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -476,12 +476,6 @@ def add_battery_constraints(n): def add_RES_constraints(n, res_share, config): - lgrouper = n.loads.bus.map(n.buses.country) - # TODO drop load - ggrouper = n.generators.bus.map(n.buses.country) - sgrouper = n.storage_units.bus.map(n.buses.country) - cgrouper = n.links.bus0.map(n.buses.country) - logger.warning( "The add_RES_constraints() is still work in progress. " "Unexpected results might be incurred, particularly if " @@ -489,31 +483,39 @@ def add_RES_constraints(n, res_share, config): "is subject to future improvements." ) - load = ( - n.snapshot_weightings.generators - @ n.loads_t.p_set.groupby(lgrouper, axis=1).sum() - ) - - rhs = res_share * load - renew_techs = config["electricity"]["renewable_carriers"] charger = ["H2 electrolysis", "battery charger"] discharger = ["H2 fuel cell", "battery discharger"] - gens_i = n.generators.query("carrier in @renew_techs").index - stores_i = n.storage_units.query("carrier in @renew_techs").index + ren_gen = n.generators.query("carrier in @renew_techs") + ren_stores = n.storage_units.query("carrier in @renew_techs") + ren_charger = n.links.query("carrier in @charger") + ren_discharger = n.links.query("carrier in @discharger") - charger_i = n.links.query("carrier in @charger").index - discharger_i = n.links.query("carrier in @discharger").index + gens_i = ren_gen.index + stores_i = ren_stores.index + charger_i = ren_charger.index + discharger_i = ren_discharger.index stores_t_weights = n.snapshot_weightings.stores + lgrouper = n.loads.bus.map(n.buses.country) + ggrouper = ren_gen.bus.map(n.buses.country) + sgrouper = ren_stores.bus.map(n.buses.country) + cgrouper = ren_charger.bus0.map(n.buses.country) + dgrouper = ren_discharger.bus0.map(n.buses.country) + + load = ( + n.snapshot_weightings.generators + @ n.loads_t.p_set.groupby(lgrouper, axis=1).sum() + ) + rhs = res_share * load + # Generators - # TODO restore grouping by countries un-commenting calls of groupby() lhs_gen = ( (n.model["Generator-p"].loc[:, gens_i] * n.snapshot_weightings.generators) - # .groupby(ggrouper.to_xarray()) + .groupby(ggrouper.to_xarray()) .sum() ) @@ -529,31 +531,15 @@ def add_RES_constraints(n, res_share, config): lambda r: r * n.links.loc[discharger_i].efficiency ) - lhs_dispatch = ( - store_disp_expr - # .groupby(sgrouper) - .sum() - ) - lhs_store = ( - store_expr - # .groupby(sgrouper) - .sum() - ) + lhs_dispatch = store_disp_expr.groupby(sgrouper).sum() + lhs_store = store_expr.groupby(sgrouper).sum() # Stores (or their resp. Link components) # Note that the variables "p0" and "p1" currently do not exist. # Thus, p0 and p1 must be derived from "p" (which exists), taking into account the link efficiency. - lhs_charge = ( - charge_expr - # .groupby(cgrouper) - .sum() - ) + lhs_charge = charge_expr.groupby(cgrouper).sum() - lhs_discharge = ( - discharge_expr - # .groupby(cgrouper) - .sum() - ) + lhs_discharge = discharge_expr.groupby(cgrouper).sum() lhs = lhs_gen + lhs_dispatch - lhs_store - lhs_charge + lhs_discharge From 25a9b55a6290ee586ffa1ec7356d6fdada642d36 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 20 Nov 2024 23:46:41 +0100 Subject: [PATCH 26/68] Add missed import --- scripts/solve_network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 41b5f42cb..2959c3d04 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -85,6 +85,7 @@ import numpy as np import pandas as pd import pypsa +import xarray as xr from _helpers import configure_logging, create_logger, override_component_attrs from linopy import merge from pypsa.descriptors import get_switchable_as_dense as get_as_dense From b6343e2de887875fc4366210399df80cb3c51448 Mon Sep 17 00:00:00 2001 From: ekatef Date: Thu, 21 Nov 2024 00:29:59 +0100 Subject: [PATCH 27/68] Fix refuse aligning dimensions --- scripts/solve_network.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 2959c3d04..be04cff17 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -428,9 +428,7 @@ def update_capacity_constraint(n): rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0) - n.model.add_constraints( - lhs <= rhs, name="gen_updated_capacity_constraint", mask=rhs.notnull() - ) + n.model.add_constraints(lhs <= rhs, name="gen_updated_capacity_constraint") def add_operational_reserve_margin(n, sns, config): From 129db46e6c3c63f348fc0ff750a9eaf040d29974 Mon Sep 17 00:00:00 2001 From: ekatef Date: Thu, 21 Nov 2024 00:30:29 +0100 Subject: [PATCH 28/68] Improve naming --- scripts/solve_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index be04cff17..309569ec6 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -423,7 +423,9 @@ def update_capacity_constraint(n): ) if not ext_i.empty: - capacity_variable = n.model["Generator-p_nom"] + capacity_variable = n.model["Generator-p_nom"].rename( + {"Generator-ext": "Generator"} + ) lhs = dispatch + reserve - capacity_variable * xr.DataArray(p_max_pu[ext_i]) rhs = (p_max_pu[fix_i] * capacity_fixed).reindex(columns=gen_i, fill_value=0) From 6770144fab547cbea66c59a4adce243691ed21f4 Mon Sep 17 00:00:00 2001 From: ekatef Date: Thu, 21 Nov 2024 00:52:46 +0100 Subject: [PATCH 29/68] Remove outdated n.variables --- scripts/solve_network.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 309569ec6..f253366f3 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -462,8 +462,7 @@ def add_battery_constraints(n): 1 * charger_size - efficiency * discharger_size = 0 """ nodes = n.buses.index[n.buses.carrier == "battery"] - # TODO Check if the second part of the condition can make sense - # if nodes.empty or ("Link", "p_nom") not in n.variables.index: + if nodes.empty: return vars_link = n.model["Link-p_nom"] @@ -608,7 +607,7 @@ def _add_land_use_constraint_m(n): def add_h2_network_cap(n, cap): h2_network = n.links.loc[n.links.carrier == "H2 pipeline"] - if h2_network.index.empty or ("Link", "p_nom") not in n.variables.index: + if h2_network.index.empty: return h2_network_cap = get_var(n, "Link", "p_nom") subset_index = h2_network.index.intersection(h2_network_cap.index) @@ -817,7 +816,7 @@ def add_chp_constraints(n): def add_co2_sequestration_limit(n, sns): co2_stores = n.stores.loc[n.stores.carrier == "co2 stored"].index - if co2_stores.empty or ("Store", "e") not in n.variables.index: + if co2_stores.empty: return vars_final_co2_stored = get_var(n, "Store", "e").loc[sns[-1], co2_stores] From aa92c9abc7e5f9ae1856b2912402d3930dcc905e Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 21 Nov 2024 16:35:06 +0100 Subject: [PATCH 30/68] adjust warning for h2_cap --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 084652a02..15f04023a 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -629,7 +629,7 @@ def add_h2_network_cap(n, cap): diff_index = h2_network_cap_index.difference(subset_index) if len(diff_index) > 0: logger.warning( - f"Impossible to set H2 cap for the following links: {diff_index}" + f"Impossible to set a limit for H2 pipelines extension for the following links: {diff_index}" ) lhs = ( h2_network_cap.loc[subset_index] * h2_network.loc[subset_index, "length"] From 1ba8d94bf51d8ba5c7c37cf6461179717a29f756 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 21 Nov 2024 16:35:52 +0100 Subject: [PATCH 31/68] revert wildcards rules --- scripts/solve_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 15f04023a..6e848c719 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1050,9 +1050,9 @@ def solve_network(n, config, solving, **kwargs): snakemake = mock_snakemake( "solve_network", simpl="", - clusters="10", + clusters="54", ll="copt", - opts="Co2L-3H", + opts="Co2L-1H", ) configure_logging(snakemake) From e909f3dfb67b654dc540b35f7027768ad0eaa59d Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 21 Nov 2024 16:37:58 +0100 Subject: [PATCH 32/68] remove comment for .variables --- scripts/solve_network.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6e848c719..aad3f6471 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -462,7 +462,6 @@ def add_battery_constraints(n): """ nodes = n.buses.index[n.buses.carrier == "battery"] # TODO Check if the second part of the condition can make sense - # if nodes.empty or ("Link", "p_nom") not in n.variables.index: if nodes.empty: return vars_link = n.model["Link-p_nom"] @@ -827,7 +826,7 @@ def add_chp_constraints(n): def add_co2_sequestration_limit(n, sns): co2_stores = n.stores.loc[n.stores.carrier == "co2 stored"].index - if co2_stores.empty: # or ("Store", "e") not in n.variables.index: + if co2_stores.empty: return vars_final_co2_stored = n.model["Store-e"].loc[sns[-1], co2_stores] From ffdf4a420b214339587aa8ae1bed9096793613c8 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 21 Dec 2024 00:16:24 +0100 Subject: [PATCH 33/68] Remove outdated dependency hint --- envs/environment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 5228000cc..a283f802c 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -92,4 +92,3 @@ dependencies: - tsam>=1.1.0 - chaospy # lastest version only available on pip - fake_useragent - #- countrycode==0.4.0 # a sub-dependency; the restriction is needed to deal with an encoding error for windows From 95d7a294715901c6eb1787b926747edfd0f5ab4c Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 21 Dec 2024 00:17:09 +0100 Subject: [PATCH 34/68] Update comments --- scripts/solve_network.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 5fc630b59..8750ee17f 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -266,7 +266,9 @@ def add_EQ_constraints(n, o, scaling=1e-1): .sum() .sum("snapshot") ) - # TODO: double check that this is really needed, why do have to subtract the spillage + # the current formulation implies that the available hydro power is (inflow - spillage) + # it implies efficiency_dispatch is 1 which is not quite general + # see https://github.com/pypsa-meets-earth/pypsa-earth/issues/1245 for possible improvements if not n.storage_units_t.inflow.empty: spillage_variable = n.model["StorageUnit-spill"] lhs_spill = ( @@ -1053,7 +1055,6 @@ def solve_network(n, config, solving, **kwargs): else: n = pypsa.Network(snakemake.input.network) - # TODO Double-check handling the augmented case if snakemake.params.augmented_line_connection.get("add_to_snakefile"): n.lines.loc[n.lines.index.str.contains("new"), "s_nom_min"] = ( snakemake.params.augmented_line_connection.get("min_expansion") From ff939d40d8588fdfeebdb175965b2356fc1eacbf Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 21 Dec 2024 00:17:45 +0100 Subject: [PATCH 35/68] Add a docstring --- scripts/solve_network.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 8750ee17f..d53256155 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -477,6 +477,17 @@ def add_battery_constraints(n): def add_RES_constraints(n, res_share, config): + """ + The constraint ensures that a predefined share of power is generated + by renewable sources + + Parameters + ---------- + n : pypsa.Network + res_share: float + config : dict + """ + logger.warning( "The add_RES_constraints() is still work in progress. " "Unexpected results might be incurred, particularly if " From 81721127d45caa4b54d175ec34efb70e44d8d8bc Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 21 Dec 2024 00:24:58 +0100 Subject: [PATCH 36/68] Update the release notes --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4a78bb4c6..f6d062413 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -15,6 +15,7 @@ This part of documentation collects descriptive release notes to capture the mai * Include option in the config to allow for custom airport data `PR #1241 `__ +* Implement changes in processing network topology to use the updated PyPSA version. `PR #1065 `__ **Minor Changes and bug-fixing** @@ -115,7 +116,6 @@ PyPSA-Earth 0.4.0 * Add an option to use csv format for custom demand imports. `PR #995 `__ -* Implement changes in processing network topology to use the updated PyPSA version. `PR #1065 `__ **Minor Changes and bug-fixing** From 6ca41778ea5f89eacb2fcb08c5228efb73ca31da Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Sat, 21 Dec 2024 00:29:01 +0100 Subject: [PATCH 37/68] Update scripts/solve_network.py Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index d53256155..7574d2275 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -349,7 +349,7 @@ def add_SAFE_constraints(n, config): p_nom = n.model["Generator-p_nom"].loc[ext_gens_i] lhs = p_nom.sum() exist_conv_caps = n.generators.query( - "~p_nom_extendable & carrier in @entional_carriers" + "~p_nom_extendable & carrier in @conventional_carriers" ).p_nom.sum() rhs = reserve_margin - exist_conv_caps n.model.add_constraints(lhs >= rhs, name="safe_mintotalcap") From eafc486b7c797f8216307489e4837e644d6b56f2 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 21 Dec 2024 15:40:16 +0100 Subject: [PATCH 38/68] Add a workflow to update environments in PR --- .github/workflows/update-pinned-env-pr.yml | 82 ++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/update-pinned-env-pr.yml diff --git a/.github/workflows/update-pinned-env-pr.yml b/.github/workflows/update-pinned-env-pr.yml new file mode 100644 index 000000000..67e2a4b10 --- /dev/null +++ b/.github/workflows/update-pinned-env-pr.yml @@ -0,0 +1,82 @@ +name: Update pinned envs + +on: + push: + paths: + - envs/environment.yaml + workflow_dispatch: + + +jobs: + update-pinned-environment-pr: + # if: ${{ github.ref == 'refs/heads/main' }} + name: Update pinned envs PR + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu, macos, windows] + include: + - os: ubuntu + suffix: "linux" + - os: macos + suffix: "macos" + - os: windows + suffix: "windows" + + steps: + - uses: actions/checkout@v4 + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ${{ github.event.repository.name }} + environment-file: envs/environment.yaml + + - name: Update pinned environment per OS + run: | + conda env export --name ${{ github.event.repository.name }} --no-builds > envs/${{ matrix.suffix }}-pinned.yaml + + - name: Add SPDX header + if: ${{ matrix.suffix != 'windows' }} + run: | + SPDX_HEADER="# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors\n#\n# SPDX-License-Identifier: CC0-1.0\n" + echo -e "$SPDX_HEADER" | cat - envs/${{ matrix.suffix }}-pinned.yaml > temp && mv temp envs/${{ matrix.suffix }}-pinned.yaml + + - name: Add SPDX header (windows) + if: ${{ matrix.suffix == 'windows' }} + run: | + $SPDX_HEADER = "# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur`r`n#`r`n# SPDX-License-Identifier: CC0-1.0`r`n`r`n" + $CurrentContent = Get-Content "envs/${{ matrix.suffix }}-pinned.yaml" -Raw + $NewContent = $SPDX_HEADER + $CurrentContent + $NewContent | Set-Content "envs/${{ matrix.suffix }}-pinned.yaml" + + - name: Cache environment files + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.suffix }}-pinned + path: envs/${{ matrix.suffix }}-pinned.yaml + + create-pull-request: + needs: update-pinned-environment + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Download all artifacts + uses: actions/download-artifact@v4 + + - name: Prepare files for commit + run: | + mkdir -p envs + mv linux-pinned/* envs/linux-pinned.yaml + mv macos-pinned/* envs/macos-pinned.yaml + mv windows-pinned/* envs/windows-pinned.yaml + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: update-pinned-environment + title: "[github-actions.ci] Update pinned envs" + body: "Automatically generated PR to update pinned environment files for Windows, macOS, and Linux." + commit-message: "Update pinned environment files for all platforms" From 3ff85e7e4741caae90368de044d617bec318aaf2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 21 Dec 2024 20:28:14 +0100 Subject: [PATCH 39/68] [github-actions.ci] Update pinned envs (#1253) * Update pinned environment files for all platforms --------- Co-authored-by: ekatef <30229437+ekatef@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- envs/linux-pinned.yaml | 101 ++++++++++++++++++++------------------- envs/macos-pinned.yaml | 92 ++++++++++++++++++----------------- envs/windows-pinned.yaml | 95 +++++++++++++++++++----------------- 3 files changed, 154 insertions(+), 134 deletions(-) diff --git a/envs/linux-pinned.yaml b/envs/linux-pinned.yaml index fadd8a2fe..38cbb18fc 100644 --- a/envs/linux-pinned.yaml +++ b/envs/linux-pinned.yaml @@ -27,16 +27,16 @@ dependencies: - atk-1.0=2.38.0 - atlite=0.3.0 - attr=2.5.1 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -50,24 +50,24 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 @@ -92,23 +92,24 @@ dependencies: - contourpy=1.3.1 - country_converter=1.2 - cpp-expected=1.1.0 +- cppad=20240000.7 - cycler=0.12.1 - cyrus-sasl=2.1.27 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 - dbus=1.13.6 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 - entrypoints=0.4 @@ -120,7 +121,7 @@ dependencies: - filelock=3.16.1 - fiona=1.9.6 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -133,7 +134,7 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - gdk-pixbuf=2.42.12 @@ -144,8 +145,8 @@ dependencies: - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - gettext=0.22.5 - gettext-tools=0.22.5 - gflags=2.2.2 @@ -174,7 +175,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -184,9 +185,9 @@ dependencies: - importlib_resources=6.4.5 - inflate64=1.0.0 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 - jinja2=3.1.4 @@ -200,10 +201,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -229,12 +230,12 @@ dependencies: - libbrotlienc=1.1.0 - libcap=2.71 - libcblas=3.9.0 -- libclang-cpp19.1=19.1.5 -- libclang13=19.1.5 +- libclang-cpp19.1=19.1.6 +- libclang13=19.1.6 - libcrc32c=1.1.2 - libcups=2.3.3 - libcurl=8.11.1 -- libdeflate=1.22 +- libdeflate=1.23 - libdrm=2.4.124 - libedit=3.1.20191231 - libegl=1.7.0 @@ -282,14 +283,14 @@ dependencies: - liblapack=3.9.0 - liblapacke=3.9.0 - libllvm14=14.0.6 -- libllvm19=19.1.5 +- libllvm19=19.1.6 - liblzma=5.6.3 - liblzma-devel=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libnghttp2=1.64.0 - libnsl=2.0.1 -- libntlm=1.4 +- libntlm=1.8 - libogg=1.3.5 - libopenblas=0.3.28 - libopus=1.3.1 @@ -318,7 +319,7 @@ dependencies: - libutf8proc=2.9.0 - libuuid=2.38.1 - libvorbis=1.3.7 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libxcb=1.17.0 - libxcrypt=4.4.36 - libxkbcommon=1.7.0 @@ -334,7 +335,7 @@ dependencies: - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -349,6 +350,7 @@ dependencies: - metis=5.1.0 - minizip=4.0.7 - mistune=3.0.2 +- mpfr=4.2.1 - mpg123=1.32.9 - msgpack-python=1.1.0 - multipledispatch=0.6.0 @@ -359,7 +361,7 @@ dependencies: - munkres=1.1.4 - mysql-common=9.0.1 - mysql-libs=9.0.1 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - ncurses=6.5 @@ -385,9 +387,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -430,13 +432,14 @@ dependencies: - pydoe2=1.3.0 - pygments=2.18.0 - pyogrio=0.10.0 -- pyomo=6.8.2 +- pyomo=6.6.1 - pyparsing=3.2.0 - pyppmd=1.1.0 - pyproj=3.7.0 -- pypsa=0.24.0 +- pypsa=0.28.0 - pyqt=5.15.9 - pyqt5-sip=12.12.2 +- pyscipopt=5.2.1 - pyshp=2.3.1 - pysocks=1.7.1 - pytables=3.10.1 @@ -470,18 +473,19 @@ dependencies: - rtree=1.3.0 - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 -- s2n=1.5.9 +- s2n=1.5.10 - scikit-learn=1.6.0 +- scip=9.2.0 - scipy=1.14.1 - seaborn=0.13.2 - seaborn-base=0.13.2 - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - sip=6.7.12 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -495,12 +499,13 @@ dependencies: - statsmodels=0.14.4 - stopit=1.1.2 - tabulate=0.9.0 +- tbb=2022.0.0 - tblib=3.0.0 - terminado=0.18.1 - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - toml=0.10.2 @@ -540,16 +545,16 @@ dependencies: - xerces-c=3.2.5 - xkeyboard-config=2.43 - xlrd=2.0.1 -- xorg-libice=1.1.1 -- xorg-libsm=1.2.4 +- xorg-libice=1.1.2 +- xorg-libsm=1.2.5 - xorg-libx11=1.8.10 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdamage=1.1.6 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - xorg-libxfixes=6.0.1 -- xorg-libxrender=0.9.11 -- xorg-libxxf86vm=1.1.5 +- xorg-libxrender=0.9.12 +- xorg-libxxf86vm=1.1.6 - xorg-xf86vidmodeproto=2.3.1 - xyzservices=2024.9.0 - xz=5.6.3 @@ -571,7 +576,7 @@ dependencies: - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 - - protobuf==5.29.1 + - highspy==1.9.0 + - protobuf==5.29.2 - tsam==2.3.6 prefix: /usr/share/miniconda/envs/pypsa-earth diff --git a/envs/macos-pinned.yaml b/envs/macos-pinned.yaml index 96e4a8c09..d872f920c 100644 --- a/envs/macos-pinned.yaml +++ b/envs/macos-pinned.yaml @@ -24,16 +24,16 @@ dependencies: - async-lru=2.0.4 - atk-1.0=2.38.0 - atlite=0.3.0 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -47,24 +47,24 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 @@ -89,22 +89,23 @@ dependencies: - contourpy=1.3.1 - country_converter=1.2 - cpp-expected=1.1.0 +- cppad=20240000.7 - cycler=0.12.1 - cyrus-sasl=2.1.27 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 - entrypoints=0.4 @@ -115,7 +116,7 @@ dependencies: - filelock=3.16.1 - fiona=1.9.6 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -128,7 +129,7 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - gdk-pixbuf=2.42.12 @@ -139,8 +140,8 @@ dependencies: - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - gflags=2.2.2 - giflib=5.2.2 - gitdb=4.0.11 @@ -155,7 +156,7 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=9.0.0 +- harfbuzz=10.1.0 - hdf4=4.2.15 - hdf5=1.14.3 - holoviews=1.20.0 @@ -163,7 +164,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -173,9 +174,9 @@ dependencies: - importlib_resources=6.4.5 - inflate64=1.0.0 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 - jinja2=3.1.4 @@ -189,10 +190,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -214,8 +215,8 @@ dependencies: - libcblas=3.9.0 - libcrc32c=1.1.2 - libcurl=8.11.1 -- libcxx=19.1.5 -- libdeflate=1.22 +- libcxx=19.1.6 +- libdeflate=1.23 - libedit=3.1.20191231 - libev=4.33 - libevent=2.1.12 @@ -242,6 +243,7 @@ dependencies: - libgoogle-cloud=2.32.0 - libgoogle-cloud-storage=2.32.0 - libgrpc=1.67.1 +- libhwloc=2.11.2 - libiconv=1.17 - libintl=0.22.5 - libjpeg-turbo=3.0.0 @@ -250,10 +252,10 @@ dependencies: - liblapacke=3.9.0 - libllvm14=14.0.6 - liblzma=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libnghttp2=1.64.0 -- libntlm=1.4 +- libntlm=1.8 - libopenblas=0.3.28 - libparquet=18.1.0 - libpng=1.6.44 @@ -272,7 +274,7 @@ dependencies: - libthrift=0.21.0 - libtiff=4.7.0 - libutf8proc=2.9.0 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libxcb=1.17.0 - libxml2=2.13.5 - libxslt=1.1.39 @@ -280,14 +282,14 @@ dependencies: - libzlib=1.3.1 - linkify-it-py=2.0.3 - linopy=0.3.11 -- llvm-openmp=19.1.5 +- llvm-openmp=19.1.6 - llvmlite=0.43.0 - locket=1.0.0 - lxml=5.3.0 - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -302,6 +304,7 @@ dependencies: - metis=5.1.0 - minizip=4.0.7 - mistune=3.0.2 +- mpfr=4.2.1 - msgpack-python=1.1.0 - multipledispatch=0.6.0 - multiurl=0.3.3 @@ -309,7 +312,7 @@ dependencies: - mumps-include=5.7.3 - mumps-seq=5.7.3 - munkres=1.1.4 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - ncurses=6.5 @@ -334,9 +337,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -380,11 +383,12 @@ dependencies: - pyobjc-core=10.3.2 - pyobjc-framework-cocoa=10.3.2 - pyogrio=0.10.0 -- pyomo=6.8.2 +- pyomo=6.6.1 - pyparsing=3.2.0 - pyppmd=1.1.0 - pyproj=3.7.0 -- pypsa=0.24.0 +- pypsa=0.28.0 +- pyscipopt=5.2.1 - pyshp=2.3.1 - pysocks=1.7.1 - pytables=3.10.1 @@ -418,15 +422,16 @@ dependencies: - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 - scikit-learn=1.6.0 +- scip=9.2.0 - scipy=1.14.1 - seaborn=0.13.2 - seaborn-base=0.13.2 - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -440,12 +445,13 @@ dependencies: - statsmodels=0.14.4 - stopit=1.1.2 - tabulate=0.9.0 +- tbb=2022.0.0 - tblib=3.0.0 - terminado=0.18.1 - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - tomli=2.2.1 @@ -478,7 +484,7 @@ dependencies: - xarray=2023.11.0 - xerces-c=3.2.5 - xlrd=2.0.1 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdmcp=1.1.5 - xyzservices=2024.9.0 - yaml=0.2.5 @@ -497,7 +503,7 @@ dependencies: - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 - - protobuf==5.29.1 + - highspy==1.9.0 + - protobuf==5.29.2 - tsam==2.3.6 prefix: /Users/runner/miniconda3/envs/pypsa-earth diff --git a/envs/windows-pinned.yaml b/envs/windows-pinned.yaml index ca45bea29..b3109fb5c 100644 --- a/envs/windows-pinned.yaml +++ b/envs/windows-pinned.yaml @@ -14,6 +14,7 @@ channels: dependencies: - _openmp_mutex=4.5 - affine=2.4.0 +- ampl-asl=1.0.0 - amply=0.1.6 - anyio=4.7.0 - appdirs=1.4.4 @@ -23,16 +24,16 @@ dependencies: - asttokens=3.0.0 - async-lru=2.0.4 - atlite=0.3.0 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -45,24 +46,24 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 @@ -81,22 +82,23 @@ dependencies: - contourpy=1.3.1 - country_converter=1.2 - cpp-expected=1.1.0 +- cppad=20240000.7 - cpython=3.10.16 - cycler=0.12.1 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 - entrypoints=0.4 @@ -107,7 +109,7 @@ dependencies: - filelock=3.16.1 - fiona=1.9.6 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -120,7 +122,7 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - geographiclib=2.0 @@ -130,14 +132,15 @@ dependencies: - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - getopt-win32=0.1 - gitdb=4.0.11 - gitpython=3.1.43 - glib=2.82.2 - glib-tools=2.82.2 - glpk=5.0 +- gmp=6.3.0 - graphite2=1.3.13 - graphviz=12.0.0 - gst-plugins-base=1.24.7 @@ -146,7 +149,7 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=9.0.0 +- harfbuzz=10.1.0 - hdf4=4.2.15 - hdf5=1.14.3 - holoviews=1.20.0 @@ -154,7 +157,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -164,9 +167,9 @@ dependencies: - importlib_resources=6.4.5 - inflate64=1.0.0 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 - jinja2=3.1.4 @@ -179,10 +182,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -198,14 +201,15 @@ dependencies: - libarrow-dataset=18.1.0 - libarrow-substrait=18.1.0 - libblas=3.9.0 +- libboost=1.86.0 - libbrotlicommon=1.1.0 - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 -- libclang13=19.1.5 +- libclang13=19.1.6 - libcrc32c=1.1.2 - libcurl=8.11.1 -- libdeflate=1.22 +- libdeflate=1.23 - libevent=2.1.12 - libexpat=2.6.4 - libffi=3.4.2 @@ -231,6 +235,7 @@ dependencies: - libgoogle-cloud=2.32.0 - libgoogle-cloud-storage=2.32.0 - libgrpc=1.67.1 +- libhwloc=2.11.2 - libiconv=1.17 - libintl=0.22.5 - libintl-devel=0.22.5 @@ -238,7 +243,7 @@ dependencies: - libkml=1.3.0 - liblapack=3.9.0 - liblzma=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libogg=1.3.5 - libopenblas=0.3.28 @@ -258,7 +263,7 @@ dependencies: - libtiff=4.7.0 - libutf8proc=2.9.0 - libvorbis=1.3.7 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libwinpthread=12.0.0.r4.gg4f2fc60ca - libxcb=1.17.0 - libxml2=2.13.5 @@ -274,7 +279,7 @@ dependencies: - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -288,13 +293,14 @@ dependencies: - mercantile=1.2.1 - minizip=4.0.7 - mistune=3.0.2 +- mpfr=4.2.1 - msgpack-python=1.1.0 - multipledispatch=0.6.0 - multiurl=0.3.3 - multivolumefile=0.2.3 - mumps-seq=5.7.3 - munkres=1.1.4 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - nest-asyncio=1.6.0 @@ -317,9 +323,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -358,14 +364,15 @@ dependencies: - pydoe2=1.3.0 - pygments=2.18.0 - pyogrio=0.10.0 -- pyomo=6.8.2 +- pyomo=6.6.1 - pyparsing=3.2.0 - pyppmd=1.1.0 - pyproj=3.7.0 -- pypsa=0.24.0 +- pypsa=0.28.0 - pyqt=5.15.9 - pyqt5-sip=12.12.2 - pyreadline3=3.5.4 +- pyscipopt=5.2.1 - pyshp=2.3.1 - pysocks=1.7.1 - pytables=3.10.1 @@ -401,16 +408,17 @@ dependencies: - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 - scikit-learn=1.6.0 +- scip=9.2.0 - scipy=1.14.1 - seaborn=0.13.2 - seaborn-base=0.13.2 - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - sip=6.7.12 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -424,12 +432,13 @@ dependencies: - statsmodels=0.14.4 - stopit=1.1.2 - tabulate=0.9.0 +- tbb=2022.0.0 - tblib=3.0.0 - terminado=0.18.1 - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - toml=0.10.2 @@ -468,10 +477,10 @@ dependencies: - xarray=2023.11.0 - xerces-c=3.2.5 - xlrd=2.0.1 -- xorg-libice=1.1.1 -- xorg-libsm=1.2.4 +- xorg-libice=1.1.2 +- xorg-libsm=1.2.5 - xorg-libx11=1.8.10 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - xorg-libxpm=3.5.17 @@ -493,8 +502,8 @@ dependencies: - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 + - highspy==1.9.0 - polars==1.17.1 - - protobuf==5.29.1 + - protobuf==5.29.2 - tsam==2.3.6 prefix: C:\Miniconda\envs\pypsa-earth From 36f45f07ce6154cef5aaddad72cfe01a7273bf56 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 21 Dec 2024 21:38:09 +0100 Subject: [PATCH 40/68] Remove an outdated dependency --- scripts/solve_network.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 2464d20e1..fde668f2b 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -91,7 +91,6 @@ from pypsa.descriptors import get_switchable_as_dense as get_as_dense from pypsa.optimization.abstract import optimize_transmission_expansion_iteratively from pypsa.optimization.optimize import optimize -from vresutils.benchmark import memory_logger logger = create_logger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) From 25f6d7369e56b4839b99e4fd9ec037a783618c39 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 21 Dec 2024 21:38:36 +0100 Subject: [PATCH 41/68] Remove a redundand workflow --- .github/workflows/update-pinned-env-pr.yml | 82 ---------------------- 1 file changed, 82 deletions(-) delete mode 100644 .github/workflows/update-pinned-env-pr.yml diff --git a/.github/workflows/update-pinned-env-pr.yml b/.github/workflows/update-pinned-env-pr.yml deleted file mode 100644 index 67e2a4b10..000000000 --- a/.github/workflows/update-pinned-env-pr.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Update pinned envs - -on: - push: - paths: - - envs/environment.yaml - workflow_dispatch: - - -jobs: - update-pinned-environment-pr: - # if: ${{ github.ref == 'refs/heads/main' }} - name: Update pinned envs PR - runs-on: ${{ matrix.os }}-latest - strategy: - fail-fast: false - matrix: - os: [ubuntu, macos, windows] - include: - - os: ubuntu - suffix: "linux" - - os: macos - suffix: "macos" - - os: windows - suffix: "windows" - - steps: - - uses: actions/checkout@v4 - - - name: Setup conda - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: ${{ github.event.repository.name }} - environment-file: envs/environment.yaml - - - name: Update pinned environment per OS - run: | - conda env export --name ${{ github.event.repository.name }} --no-builds > envs/${{ matrix.suffix }}-pinned.yaml - - - name: Add SPDX header - if: ${{ matrix.suffix != 'windows' }} - run: | - SPDX_HEADER="# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors\n#\n# SPDX-License-Identifier: CC0-1.0\n" - echo -e "$SPDX_HEADER" | cat - envs/${{ matrix.suffix }}-pinned.yaml > temp && mv temp envs/${{ matrix.suffix }}-pinned.yaml - - - name: Add SPDX header (windows) - if: ${{ matrix.suffix == 'windows' }} - run: | - $SPDX_HEADER = "# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur`r`n#`r`n# SPDX-License-Identifier: CC0-1.0`r`n`r`n" - $CurrentContent = Get-Content "envs/${{ matrix.suffix }}-pinned.yaml" -Raw - $NewContent = $SPDX_HEADER + $CurrentContent - $NewContent | Set-Content "envs/${{ matrix.suffix }}-pinned.yaml" - - - name: Cache environment files - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.suffix }}-pinned - path: envs/${{ matrix.suffix }}-pinned.yaml - - create-pull-request: - needs: update-pinned-environment - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Download all artifacts - uses: actions/download-artifact@v4 - - - name: Prepare files for commit - run: | - mkdir -p envs - mv linux-pinned/* envs/linux-pinned.yaml - mv macos-pinned/* envs/macos-pinned.yaml - mv windows-pinned/* envs/windows-pinned.yaml - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: update-pinned-environment - title: "[github-actions.ci] Update pinned envs" - body: "Automatically generated PR to update pinned environment files for Windows, macOS, and Linux." - commit-message: "Update pinned environment files for all platforms" From 0f1d8ce2673a0788c0ca963c75a53338bd8ccff1 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 22 Dec 2024 10:49:11 +0100 Subject: [PATCH 42/68] Get back CI for Windows --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a243a304b..8ab61e1e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,8 +31,8 @@ jobs: env_file: envs/linux-pinned.yaml - os: macos env_file: envs/macos-pinned.yaml - # - os: windows - # env_file: envs/windows-pinned.yaml + - os: windows + env_file: envs/windows-pinned.yaml defaults: run: From 00f98ba0326d657dd8642d417e88b08c815dca0c Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Mon, 23 Dec 2024 21:28:06 +0100 Subject: [PATCH 43/68] Add env fix --- envs/environment.yaml | 4 ++-- scripts/build_shapes.py | 2 +- scripts/cluster_network.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 96278f6f2..dd5670196 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -33,8 +33,8 @@ dependencies: # starting from 1.3.5 numpoly requires numpy>2.0 which leads to issues - numpoly<=1.3.4 - pandas -- geopandas>=0.11.0 -- fiona!=1.8.22 +- geopandas>=1 +- fiona>=1.10 - xarray>=2023.11.0, <2023.12.0 - netcdf4 - networkx diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 3e0e8bd5a..a09829302 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -315,7 +315,7 @@ def save_to_geojson(df, fn): if df.shape[0] > 0: df = df.reset_index() schema = {**gpd.io.file.infer_schema(df), "geometry": "Unknown"} - df.to_file(fn, driver="GeoJSON", schema=schema) + df.to_file(fn, driver="GeoJSON", schema=schema, engine="fiona") else: # create empty file to avoid issues with snakemake with open(fn, "w") as fp: diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 63ae6556c..2eae2f4d8 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -635,7 +635,7 @@ def save_to_geojson(s, fn): os.unlink(fn) df = s.reset_index() schema = {**gpd.io.file.infer_schema(df), "geometry": "Unknown"} - df.to_file(fn, driver="GeoJSON", schema=schema) + df.to_file(fn, driver="GeoJSON", schema=schema, engine="fiona") def cluster_regions(busmaps, inputs, output): From 255aa605f590d4e625212e9c79b9ac2b8f6b8c8c Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Mon, 23 Dec 2024 21:46:13 +0100 Subject: [PATCH 44/68] Squashed commit of the following: commit a220390988cceb189a06906e3384c466d846f521 Author: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon Dec 23 19:39:49 2024 +0000 [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci commit 6bff9547a36847fdba98b6bb0aff5ad200f0508d Author: davide-f <67809479+davide-f@users.noreply.github.com> Date: Mon Dec 23 19:39:35 2024 +0000 Update pinned environment files for all platforms --- envs/linux-pinned.yaml | 115 +++++++++++++++++++-------------------- envs/macos-pinned.yaml | 102 +++++++++++++++++----------------- envs/windows-pinned.yaml | 103 +++++++++++++++++------------------ 3 files changed, 157 insertions(+), 163 deletions(-) diff --git a/envs/linux-pinned.yaml b/envs/linux-pinned.yaml index fadd8a2fe..9faa2139b 100644 --- a/envs/linux-pinned.yaml +++ b/envs/linux-pinned.yaml @@ -27,16 +27,16 @@ dependencies: - atk-1.0=2.38.0 - atlite=0.3.0 - attr=2.5.1 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -50,30 +50,30 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 - cftime=1.6.4 - charset-normalizer=3.4.0 -- click=8.1.7 +- click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 - cloudpickle=3.1.0 @@ -94,23 +94,24 @@ dependencies: - cpp-expected=1.1.0 - cycler=0.12.1 - cyrus-sasl=2.1.27 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 - dbus=1.13.6 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 +- earth-osm=2.3.post1 - entrypoints=0.4 - entsoe-py=0.6.11 - et_xmlfile=2.0.0 @@ -118,9 +119,9 @@ dependencies: - executing=2.1.0 - expat=2.6.4 - filelock=3.16.1 -- fiona=1.9.6 +- fiona=1.10.1 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -133,19 +134,19 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - gdk-pixbuf=2.42.12 - geographiclib=2.0 - geojson-rewind=1.1.0 -- geopandas=0.14.3 -- geopandas-base=0.14.3 +- geopandas=1.0.1 +- geopandas-base=1.0.1 - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - gettext=0.22.5 - gettext-tools=0.22.5 - gflags=2.2.2 @@ -166,7 +167,7 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=9.0.0 +- harfbuzz=10.1.0 - hdf4=4.2.15 - hdf5=1.14.3 - holoviews=1.20.0 @@ -174,7 +175,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -182,14 +183,14 @@ dependencies: - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 - importlib_resources=6.4.5 -- inflate64=1.0.0 +- inflate64=1.0.1 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 -- jinja2=3.1.4 +- jinja2=3.1.5 - joblib=1.4.2 - json-c=0.18 - json5=0.10.0 @@ -200,10 +201,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -229,12 +230,12 @@ dependencies: - libbrotlienc=1.1.0 - libcap=2.71 - libcblas=3.9.0 -- libclang-cpp19.1=19.1.5 -- libclang13=19.1.5 +- libclang-cpp19.1=19.1.6 +- libclang13=19.1.6 - libcrc32c=1.1.2 - libcups=2.3.3 - libcurl=8.11.1 -- libdeflate=1.22 +- libdeflate=1.23 - libdrm=2.4.124 - libedit=3.1.20191231 - libegl=1.7.0 @@ -282,14 +283,14 @@ dependencies: - liblapack=3.9.0 - liblapacke=3.9.0 - libllvm14=14.0.6 -- libllvm19=19.1.5 +- libllvm19=19.1.6 - liblzma=5.6.3 - liblzma-devel=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libnghttp2=1.64.0 - libnsl=2.0.1 -- libntlm=1.4 +- libntlm=1.8 - libogg=1.3.5 - libopenblas=0.3.28 - libopus=1.3.1 @@ -305,7 +306,6 @@ dependencies: - libsndfile=1.2.2 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.0.0 - libspatialite=5.1.0 - libspral=2024.05.08 - libsqlite=3.47.2 @@ -318,7 +318,7 @@ dependencies: - libutf8proc=2.9.0 - libuuid=2.38.1 - libvorbis=1.3.7 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libxcb=1.17.0 - libxcrypt=4.4.36 - libxkbcommon=1.7.0 @@ -334,7 +334,7 @@ dependencies: - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -359,7 +359,7 @@ dependencies: - munkres=1.1.4 - mysql-common=9.0.1 - mysql-libs=9.0.1 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - ncurses=6.5 @@ -385,9 +385,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -412,6 +412,7 @@ dependencies: - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 +- protobuf=5.28.2 - psutil=6.1.0 - pthread-stubs=0.4 - ptyprocess=0.7.0 @@ -422,7 +423,7 @@ dependencies: - py7zr=0.22.0 - pyarrow=18.1.0 - pyarrow-core=18.1.0 -- pybcj=1.0.2 +- pybcj=1.0.3 - pycountry=24.6.1 - pycparser=2.22 - pycryptodomex=3.21.0 @@ -467,10 +468,9 @@ dependencies: - rfc3986-validator=0.1.1 - rioxarray=0.17.0 - rpds-py=0.22.3 -- rtree=1.3.0 - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 -- s2n=1.5.9 +- s2n=1.5.10 - scikit-learn=1.6.0 - scipy=1.14.1 - seaborn=0.13.2 @@ -478,10 +478,10 @@ dependencies: - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - sip=6.7.12 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -500,7 +500,7 @@ dependencies: - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - toml=0.10.2 @@ -522,7 +522,7 @@ dependencies: - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 -- urllib3=2.2.3 +- urllib3=2.3.0 - validators=0.34.0 - virtualenv=20.28.0 - wcwidth=0.2.13 @@ -540,17 +540,16 @@ dependencies: - xerces-c=3.2.5 - xkeyboard-config=2.43 - xlrd=2.0.1 -- xorg-libice=1.1.1 -- xorg-libsm=1.2.4 +- xorg-libice=1.1.2 +- xorg-libsm=1.2.5 - xorg-libx11=1.8.10 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdamage=1.1.6 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - xorg-libxfixes=6.0.1 -- xorg-libxrender=0.9.11 -- xorg-libxxf86vm=1.1.5 -- xorg-xf86vidmodeproto=2.3.1 +- xorg-libxrender=0.9.12 +- xorg-libxxf86vm=1.1.6 - xyzservices=2024.9.0 - xz=5.6.3 - xz-gpl-tools=5.6.3 @@ -568,10 +567,8 @@ dependencies: - zstd=1.5.6 - pip: - chaospy==4.3.17 - - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 - - protobuf==5.29.1 + - highspy==1.9.0 - tsam==2.3.6 prefix: /usr/share/miniconda/envs/pypsa-earth diff --git a/envs/macos-pinned.yaml b/envs/macos-pinned.yaml index 96e4a8c09..7c053a54c 100644 --- a/envs/macos-pinned.yaml +++ b/envs/macos-pinned.yaml @@ -24,16 +24,16 @@ dependencies: - async-lru=2.0.4 - atk-1.0=2.38.0 - atlite=0.3.0 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -47,30 +47,30 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 - cftime=1.6.4 - charset-normalizer=3.4.0 -- click=8.1.7 +- click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 - cloudpickle=3.1.0 @@ -91,31 +91,32 @@ dependencies: - cpp-expected=1.1.0 - cycler=0.12.1 - cyrus-sasl=2.1.27 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 +- earth-osm=2.3.post1 - entrypoints=0.4 - entsoe-py=0.6.11 - et_xmlfile=2.0.0 - exceptiongroup=1.2.2 - executing=2.1.0 - filelock=3.16.1 -- fiona=1.9.6 +- fiona=1.10.1 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -128,19 +129,19 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - gdk-pixbuf=2.42.12 - geographiclib=2.0 - geojson-rewind=1.1.0 -- geopandas=0.14.3 -- geopandas-base=0.14.3 +- geopandas=1.0.1 +- geopandas-base=1.0.1 - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - gflags=2.2.2 - giflib=5.2.2 - gitdb=4.0.11 @@ -155,7 +156,7 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=9.0.0 +- harfbuzz=10.1.0 - hdf4=4.2.15 - hdf5=1.14.3 - holoviews=1.20.0 @@ -163,7 +164,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -171,14 +172,14 @@ dependencies: - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 - importlib_resources=6.4.5 -- inflate64=1.0.0 +- inflate64=1.0.1 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 -- jinja2=3.1.4 +- jinja2=3.1.5 - joblib=1.4.2 - json-c=0.18 - json5=0.10.0 @@ -189,10 +190,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -214,8 +215,8 @@ dependencies: - libcblas=3.9.0 - libcrc32c=1.1.2 - libcurl=8.11.1 -- libcxx=19.1.5 -- libdeflate=1.22 +- libcxx=19.1.6 +- libdeflate=1.23 - libedit=3.1.20191231 - libev=4.33 - libevent=2.1.12 @@ -250,10 +251,10 @@ dependencies: - liblapacke=3.9.0 - libllvm14=14.0.6 - liblzma=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libnghttp2=1.64.0 -- libntlm=1.4 +- libntlm=1.8 - libopenblas=0.3.28 - libparquet=18.1.0 - libpng=1.6.44 @@ -265,14 +266,13 @@ dependencies: - libscotch=7.0.5 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.0.0 - libspatialite=5.1.0 - libsqlite=3.47.2 - libssh2=1.11.1 - libthrift=0.21.0 - libtiff=4.7.0 - libutf8proc=2.9.0 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libxcb=1.17.0 - libxml2=2.13.5 - libxslt=1.1.39 @@ -280,14 +280,14 @@ dependencies: - libzlib=1.3.1 - linkify-it-py=2.0.3 - linopy=0.3.11 -- llvm-openmp=19.1.5 +- llvm-openmp=19.1.6 - llvmlite=0.43.0 - locket=1.0.0 - lxml=5.3.0 - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -309,7 +309,7 @@ dependencies: - mumps-include=5.7.3 - mumps-seq=5.7.3 - munkres=1.1.4 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - ncurses=6.5 @@ -334,9 +334,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -361,6 +361,7 @@ dependencies: - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 +- protobuf=5.28.2 - psutil=6.1.0 - pthread-stubs=0.4 - ptyprocess=0.7.0 @@ -370,7 +371,7 @@ dependencies: - py7zr=0.22.0 - pyarrow=18.1.0 - pyarrow-core=18.1.0 -- pybcj=1.0.2 +- pybcj=1.0.3 - pycountry=24.6.1 - pycparser=2.22 - pycryptodomex=3.21.0 @@ -414,7 +415,6 @@ dependencies: - rfc3986-validator=0.1.1 - rioxarray=0.17.0 - rpds-py=0.22.3 -- rtree=1.3.0 - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 - scikit-learn=1.6.0 @@ -424,9 +424,9 @@ dependencies: - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -445,7 +445,7 @@ dependencies: - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - tomli=2.2.1 @@ -466,7 +466,7 @@ dependencies: - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 -- urllib3=2.2.3 +- urllib3=2.3.0 - validators=0.34.0 - virtualenv=20.28.0 - wcwidth=0.2.13 @@ -478,7 +478,7 @@ dependencies: - xarray=2023.11.0 - xerces-c=3.2.5 - xlrd=2.0.1 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdmcp=1.1.5 - xyzservices=2024.9.0 - yaml=0.2.5 @@ -494,10 +494,8 @@ dependencies: - zstd=1.5.6 - pip: - chaospy==4.3.17 - - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 - - protobuf==5.29.1 + - highspy==1.9.0 - tsam==2.3.6 prefix: /Users/runner/miniconda3/envs/pypsa-earth diff --git a/envs/windows-pinned.yaml b/envs/windows-pinned.yaml index ca45bea29..9d14aa2a7 100644 --- a/envs/windows-pinned.yaml +++ b/envs/windows-pinned.yaml @@ -14,6 +14,7 @@ channels: dependencies: - _openmp_mutex=4.5 - affine=2.4.0 +- ampl-asl=1.0.0 - amply=0.1.6 - anyio=4.7.0 - appdirs=1.4.4 @@ -23,16 +24,16 @@ dependencies: - asttokens=3.0.0 - async-lru=2.0.4 - atlite=0.3.0 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -45,30 +46,30 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 - cftime=1.6.4 - charset-normalizer=3.4.0 -- click=8.1.7 +- click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 - cloudpickle=3.1.0 @@ -83,31 +84,32 @@ dependencies: - cpp-expected=1.1.0 - cpython=3.10.16 - cycler=0.12.1 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 +- earth-osm=2.3.post1 - entrypoints=0.4 - entsoe-py=0.6.11 - et_xmlfile=2.0.0 - exceptiongroup=1.2.2 - executing=2.1.0 - filelock=3.16.1 -- fiona=1.9.6 +- fiona=1.10.1 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -120,18 +122,18 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - geographiclib=2.0 - geojson-rewind=1.1.0 -- geopandas=0.14.3 -- geopandas-base=0.14.3 +- geopandas=1.0.1 +- geopandas-base=1.0.1 - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - getopt-win32=0.1 - gitdb=4.0.11 - gitpython=3.1.43 @@ -146,7 +148,7 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=9.0.0 +- harfbuzz=10.1.0 - hdf4=4.2.15 - hdf5=1.14.3 - holoviews=1.20.0 @@ -154,7 +156,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -162,14 +164,14 @@ dependencies: - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 - importlib_resources=6.4.5 -- inflate64=1.0.0 +- inflate64=1.0.1 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 -- jinja2=3.1.4 +- jinja2=3.1.5 - joblib=1.4.2 - json5=0.10.0 - jsonpointer=3.0.0 @@ -179,10 +181,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -202,10 +204,10 @@ dependencies: - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 -- libclang13=19.1.5 +- libclang13=19.1.6 - libcrc32c=1.1.2 - libcurl=8.11.1 -- libdeflate=1.22 +- libdeflate=1.23 - libevent=2.1.12 - libexpat=2.6.4 - libffi=3.4.2 @@ -238,7 +240,7 @@ dependencies: - libkml=1.3.0 - liblapack=3.9.0 - liblzma=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libogg=1.3.5 - libopenblas=0.3.28 @@ -250,7 +252,6 @@ dependencies: - librttopo=1.1.0 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.0.0 - libspatialite=5.1.0 - libsqlite=3.47.2 - libssh2=1.11.1 @@ -258,7 +259,7 @@ dependencies: - libtiff=4.7.0 - libutf8proc=2.9.0 - libvorbis=1.3.7 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libwinpthread=12.0.0.r4.gg4f2fc60ca - libxcb=1.17.0 - libxml2=2.13.5 @@ -274,7 +275,7 @@ dependencies: - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -294,7 +295,7 @@ dependencies: - multivolumefile=0.2.3 - mumps-seq=5.7.3 - munkres=1.1.4 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - nest-asyncio=1.6.0 @@ -317,9 +318,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -342,6 +343,7 @@ dependencies: - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 +- protobuf=5.28.2 - psutil=6.1.0 - pthread-stubs=0.4 - pulp=2.7.0 @@ -350,7 +352,7 @@ dependencies: - py7zr=0.22.0 - pyarrow=18.1.0 - pyarrow-core=18.1.0 -- pybcj=1.0.2 +- pybcj=1.0.3 - pycountry=24.6.1 - pycparser=2.22 - pycryptodomex=3.21.0 @@ -397,7 +399,6 @@ dependencies: - rfc3986-validator=0.1.1 - rioxarray=0.17.0 - rpds-py=0.22.3 -- rtree=1.3.0 - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 - scikit-learn=1.6.0 @@ -407,10 +408,10 @@ dependencies: - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - sip=6.7.12 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -429,7 +430,7 @@ dependencies: - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - toml=0.10.2 @@ -451,7 +452,7 @@ dependencies: - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 -- urllib3=2.2.3 +- urllib3=2.3.0 - validators=0.34.0 - vc=14.3 - vc14_runtime=14.42.34433 @@ -468,10 +469,10 @@ dependencies: - xarray=2023.11.0 - xerces-c=3.2.5 - xlrd=2.0.1 -- xorg-libice=1.1.1 -- xorg-libsm=1.2.4 +- xorg-libice=1.1.2 +- xorg-libsm=1.2.5 - xorg-libx11=1.8.10 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - xorg-libxpm=3.5.17 @@ -490,11 +491,9 @@ dependencies: - zstd=1.5.6 - pip: - chaospy==4.3.17 - - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 + - highspy==1.9.0 - polars==1.17.1 - - protobuf==5.29.1 - tsam==2.3.6 prefix: C:\Miniconda\envs\pypsa-earth From b1f530b049a7a6d3d5f57a1c3a5a4243c34b5f6a Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Mon, 23 Dec 2024 21:54:53 +0100 Subject: [PATCH 45/68] Fixes to adapt to geopandas >=1 --- scripts/_helpers.py | 3 ++- scripts/build_bus_regions.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 36cf0d95c..e4bb7b32c 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -60,7 +60,8 @@ def check_config_version(config, fp_config=CONFIG_DEFAULT_PATH): f"The current version of 'config.yaml' doesn't match to the code version:\n\r" f" {current_config_version} provided, {actual_config_version} expected.\n\r" f"That can lead to weird errors during execution of the workflow.\n\r" - f"Please update 'config.yaml' according to 'config.default.yaml' ." + f"Please update 'config.yaml' according to 'config.default.yaml'\n\r" + "and update the environment to the latest version." ) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 1a0dc2338..21b39fc96 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -141,7 +141,7 @@ def get_gadm_shape( # when duplicates, keep only the first entry join_geos = join_geos[~join_geos.index.duplicated()] - gadm_sel = gadm_shapes.loc[join_geos["index_right"].values] + gadm_sel = gadm_shapes.loc[join_geos[gadm_shapes.index.name].values] return gadm_sel.geometry.values, gadm_sel.index.values From 9dcfd1e223364f37eaeb436b8820b7f71018ab9b Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Dec 2024 09:07:11 +0100 Subject: [PATCH 46/68] Minor refactoring --- scripts/solve_network.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index fde668f2b..2dd80b17a 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -373,14 +373,14 @@ def add_operational_reserve_margin_constraint(n, sns, config): 0, np.inf, coords=[sns, n.generators.index], name="Generator-r" ) reserve = n.model["Generator-r"] - lhs = reserve.sum("Generator") + summed_reserve = reserve.sum("Generator") # Share of extendable renewable capacities ext_i = n.generators.query("p_nom_extendable").index vres_i = n.generators_t.p_max_pu.columns if not ext_i.empty and not vres_i.empty: capacity_factor = n.generators_t.p_max_pu[vres_i.intersection(ext_i)] - renewable_capacity_variables = ( + p_nom_vres = ( n.model["Generator-p_nom"] .loc[vres_i.intersection(ext_i)] .rename({"Generator-ext": "Generator"}) @@ -472,7 +472,8 @@ def add_battery_constraints(n): # for some reasons, eff is one element longer as compared with vars_link vars_link.sel({"Link-ext": nodes + " discharger"}) * -eff[0], ) - n.model.add_constraints(lhs == 0, name="link_charger_ratio") + + n.model.add_constraints(lhs == 0, name="Link-charger_ratio") def add_RES_constraints(n, res_share, config): From 82e431f85675cac07b8e20f49c14b0b335deffae Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Dec 2024 09:07:42 +0100 Subject: [PATCH 47/68] Replace merge functions --- scripts/solve_network.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 2dd80b17a..b03a2a101 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -385,12 +385,9 @@ def add_operational_reserve_margin_constraint(n, sns, config): .loc[vres_i.intersection(ext_i)] .rename({"Generator-ext": "Generator"}) ) - lhs = merge( - lhs, - (renewable_capacity_variables * (-EPSILON_VRES * capacity_factor)).sum( - ["Generator"] - ), - ) + lhs = summed_reserve + ( + p_nom_vres * (-EPSILON_VRES * xr.DataArray(capacity_factor)), + ).sum("Generator") # Total demand per t demand = get_as_dense(n, "Load", "p_set").sum(axis=1) @@ -418,10 +415,7 @@ def update_capacity_constraint(n): p_max_pu = get_as_dense(n, "Generator", "p_max_pu") - lhs = merge( - dispatch * 1, - reserve * 1, - ) + lhs = dispatch + reserve if not ext_i.empty: capacity_variable = n.model["Generator-p_nom"].rename( @@ -462,15 +456,19 @@ def add_battery_constraints(n): Add constraint ensuring that charger = discharger, i.e. 1 * charger_size - efficiency * discharger_size = 0 """ - nodes = n.buses.index[n.buses.carrier == "battery"] - if nodes.empty: + if not n.links.p_nom_extendable.any(): return - vars_link = n.model["Link-p_nom"] - eff = n.links.loc[nodes + " discharger", "efficiency"] - lhs = merge( - vars_link.sel({"Link-ext": nodes + " charger"}) * 1, - # for some reasons, eff is one element longer as compared with vars_link - vars_link.sel({"Link-ext": nodes + " discharger"}) * -eff[0], + + discharger_bool = n.links.index.str.contains("battery discharger") + charger_bool = n.links.index.str.contains("battery charger") + + dischargers_ext = n.links[discharger_bool].query("p_nom_extendable").index + chargers_ext = n.links[charger_bool].query("p_nom_extendable").index + + eff = n.links.efficiency[dischargers_ext].values + lhs = ( + n.model["Link-p_nom"].loc[chargers_ext] + - n.model["Link-p_nom"].loc[dischargers_ext] * eff ) n.model.add_constraints(lhs == 0, name="Link-charger_ratio") From d6b36591636f1cc98448a59602e6bc0430ba5796 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Dec 2024 09:56:06 +0100 Subject: [PATCH 48/68] Add a TODO --- scripts/solve_network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index b03a2a101..32e25795e 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -417,6 +417,7 @@ def update_capacity_constraint(n): lhs = dispatch + reserve + # TODO check if `p_max_pu[ext_i]` is safe for empty `ext_i` and drop if cause in case if not ext_i.empty: capacity_variable = n.model["Generator-p_nom"].rename( {"Generator-ext": "Generator"} From 89c180bc40a27c6eb4d257efec15b9a9f1fe32e6 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Tue, 24 Dec 2024 10:14:22 +0100 Subject: [PATCH 49/68] Update text check_config_version --- scripts/_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 9b0b2d12e..8b8135063 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -60,8 +60,8 @@ def check_config_version(config, fp_config=CONFIG_DEFAULT_PATH): f"The current version of 'config.yaml' doesn't match to the code version:\n\r" f" {current_config_version} provided, {actual_config_version} expected.\n\r" f"That can lead to weird errors during execution of the workflow.\n\r" - f"Please update 'config.yaml' according to 'config.default.yaml'\n\r" - "and update the environment to the latest version." + f"Please update 'config.yaml' according to 'config.default.yaml.'\n\r" + "If issues persist, consider to update the environment to the latest version." ) From d3382a22ae8b56b4dd91bb9126d72576add1764f Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Tue, 24 Dec 2024 10:16:33 +0100 Subject: [PATCH 50/68] Revise sjoin to gpd 1.0 and drop unused variables --- scripts/build_osm_network.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index b4e7e6f4c..af1ca22a9 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -554,21 +554,17 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): buffer_df = df_p.buffer(tol).to_frame() # Spatial join to find lines intersecting point buffers - joined = gpd.sjoin(df_l, buffer_df, how="inner", op="intersects") + joined = gpd.sjoin(df_l, buffer_df, how="inner", predicate="intersects") # group lines by their index group_lines = joined.groupby(level=0) # iterate over the groups, TODO: change to apply for i, group in group_lines: - line_id = df_l.loc[i, line_id_str] # pick the line id that represents the group line_geom = df_l.loc[i, "geometry"] - # number of points that intersect with the line - num_points = len(group) - # get the indices of the points that intersect with the line - points_indexes = group["index_right"].tolist() + points_indexes = group[buffer_df.index.name].tolist() # get the geometries of the points that intersect with the line all_points = df_p.loc[points_indexes, "geometry"] From e4386fcc6611d0ccbc734514fa0749371ff844fb Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Dec 2024 10:41:47 +0100 Subject: [PATCH 51/68] Fix typo --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 32e25795e..398ca6656 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -386,7 +386,7 @@ def add_operational_reserve_margin_constraint(n, sns, config): .rename({"Generator-ext": "Generator"}) ) lhs = summed_reserve + ( - p_nom_vres * (-EPSILON_VRES * xr.DataArray(capacity_factor)), + p_nom_vres * (-EPSILON_VRES * xr.DataArray(capacity_factor)) ).sum("Generator") # Total demand per t From 4b05b025ee13ad4938d2b95bf78c61832a9e9ec5 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 28 Dec 2024 00:23:06 +0100 Subject: [PATCH 52/68] Update configuration strategies --- config.default.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config.default.yaml b/config.default.yaml index ed8554002..9a1e6ffd6 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -99,6 +99,7 @@ cluster_options: p_nom_max: sum p_nom_min: sum p_min_pu: mean + p_max_pu: weighted_average marginal_cost: mean committable: any ramp_limit_up: max From d463de7d15dd7a39821a995d949ecacc150b73d1 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Thu, 2 Jan 2025 22:45:23 +0100 Subject: [PATCH 53/68] Unify save_to_geojson definition --- scripts/build_shapes.py | 18 +----------------- scripts/cluster_network.py | 8 -------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index a09829302..a4247e64b 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -22,6 +22,7 @@ BASE_DIR, configure_logging, create_logger, + save_to_geojson, three_2_two_digits_country, two_2_three_digits_country, two_digits_2_name_country, @@ -305,23 +306,6 @@ def country_cover(country_shapes, eez_shapes=None, out_logging=False, distance=0 return africa_shape -def save_to_geojson(df, fn): - if os.path.exists(fn): - os.unlink(fn) # remove file if it exists - if not isinstance(df, gpd.GeoDataFrame): - df = gpd.GeoDataFrame(dict(geometry=df)) - - # save file if the GeoDataFrame is non-empty - if df.shape[0] > 0: - df = df.reset_index() - schema = {**gpd.io.file.infer_schema(df), "geometry": "Unknown"} - df.to_file(fn, driver="GeoJSON", schema=schema, engine="fiona") - else: - # create empty file to avoid issues with snakemake - with open(fn, "w") as fp: - pass - - def load_EEZ(countries_codes, geo_crs, EEZ_gpkg="./data/eez/eez_v11.gpkg"): """ Function to load the database of the Exclusive Economic Zones. diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 9055e0dff..9369db89a 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -634,14 +634,6 @@ def clustering_for_n_clusters( return clustering -def save_to_geojson(s, fn): - if os.path.exists(fn): - os.unlink(fn) - df = s.reset_index() - schema = {**gpd.io.file.infer_schema(df), "geometry": "Unknown"} - df.to_file(fn, driver="GeoJSON", schema=schema, engine="fiona") - - def cluster_regions(busmaps, inputs, output): busmap = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0]) From 38ac5eae9cc1a128df0638a540f9a979dad0b42b Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Fri, 3 Jan 2025 17:57:42 +0100 Subject: [PATCH 54/68] Move tsam installation to conda --- envs/environment.yaml | 2 +- envs/linux-pinned.yaml | 1 + envs/macos-pinned.yaml | 1 + envs/windows-pinned.yaml | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 9d52c1688..c336868b1 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -51,6 +51,7 @@ dependencies: - pyogrio - numba - py7zr +- tsam>=1.1.0 # Keep in conda environment when calling ipython - ipython @@ -86,6 +87,5 @@ dependencies: - pip: - git+https://github.com/davide-f/google-drive-downloader@master # google drive with fix for virus scan - - tsam>=1.1.0 - chaospy # lastest version only available on pip - fake_useragent diff --git a/envs/linux-pinned.yaml b/envs/linux-pinned.yaml index 7750d3b6d..d35941c07 100644 --- a/envs/linux-pinned.yaml +++ b/envs/linux-pinned.yaml @@ -516,6 +516,7 @@ dependencies: - tornado=6.4.2 - tqdm=4.67.1 - traitlets=5.14.3 +- tsam=2.3.6 - types-python-dateutil=2.9.0.20241206 - typing-extensions=4.12.2 - typing_extensions=4.12.2 diff --git a/envs/macos-pinned.yaml b/envs/macos-pinned.yaml index dcd6a9fbf..c6d1be082 100644 --- a/envs/macos-pinned.yaml +++ b/envs/macos-pinned.yaml @@ -461,6 +461,7 @@ dependencies: - tornado=6.4.2 - tqdm=4.67.1 - traitlets=5.14.3 +- tsam=2.3.6 - types-python-dateutil=2.9.0.20241206 - typing-extensions=4.12.2 - typing_extensions=4.12.2 diff --git a/envs/windows-pinned.yaml b/envs/windows-pinned.yaml index dfd0173b8..c43b70e60 100644 --- a/envs/windows-pinned.yaml +++ b/envs/windows-pinned.yaml @@ -449,6 +449,7 @@ dependencies: - tornado=6.4.2 - tqdm=4.67.1 - traitlets=5.14.3 +- tsam=2.3.6 - types-python-dateutil=2.9.0.20241206 - typing-extensions=4.12.2 - typing_extensions=4.12.2 @@ -504,5 +505,4 @@ dependencies: - googledrivedownloader==0.4 - highspy==1.9.0 - polars==1.17.1 - - tsam==2.3.6 prefix: C:\Miniconda\envs\pypsa-earth From 7835858a4e54653c7f52f75d8401221effd1dbdd Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Tue, 14 Jan 2025 18:31:44 +0100 Subject: [PATCH 55/68] Add release note --- doc/release_notes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 21d25acd8..022e5f411 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -17,6 +17,10 @@ This part of documentation collects descriptive release notes to capture the mai * In alternative clustering, generate hydro inflows by shape and avoid hydro inflows duplication for plants installed in the same node `PR #1120 ` +* Support of Linopy for Power and Sector-Coupled Modelling and latest PyPSA version `PR #1172 `__ + +* Update workflow to geopandas >= 1.0 `PR #1276 `__ + **Minor Changes and bug-fixing** * Prevent computation of powerplantmatching if replace option is selected for custom_powerplants `PR #1281 `__ From 60fe1d0c27579bdc2e65208a60e90e08d0fb37d8 Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 6 Jan 2025 10:47:13 +0100 Subject: [PATCH 56/68] Update the arguments list for a special case --- scripts/cluster_network.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 1f4f90353..562b9cfa7 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -699,9 +699,7 @@ def cluster_regions(busmaps, inputs, output): # Fast-path if no clustering is necessary busmap = n.buses.index.to_series() linemap = n.lines.index.to_series() - clustering = pypsa.clustering.spatial.Clustering( - n, busmap, linemap, linemap, pd.Series(dtype="O") - ) + clustering = pypsa.clustering.spatial.Clustering(n, busmap, linemap) elif len(n.buses) < n_clusters: logger.error( f"Desired number of clusters ({n_clusters}) higher than the number of buses ({len(n.buses)})" From 0a3cd173f8415fc6f2a4e596d04ce102af5a8ada Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Thu, 16 Jan 2025 01:05:50 +0100 Subject: [PATCH 57/68] Implement linopy into add_lossy_bidirectional --- scripts/solve_network.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 07ea180fc..2541ae9b2 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -933,9 +933,9 @@ def add_lossy_bidirectional_link_constraints(n: pypsa.components.Network) -> Non carriers = n.links.loc[n.links.reversed, "carrier"].unique() # noqa: F841 # get the indices of all forward links (non-reversed), that have a reversed counterpart - forward_i = n.links.loc[ - n.links.carrier.isin(carriers) & ~n.links.reversed & n.links.p_nom_extendable - ].index + forward_i = n.links.query( + "carrier in @carriers and ~reversed and p_nom_extendable" + ).index # function to get backward (reversed) indices corresponding to forward links # this function is required to properly interact with the myopic naming scheme @@ -955,11 +955,11 @@ def get_backward_i(forward_i): backward_i = get_backward_i(forward_i) # get the p_nom optimization variables for the links using the get_var function - links_p_nom = get_var(n, "Link", "p_nom") + links_p_nom = n.model["Link-p_nom"] # only consider forward and backward links that are present in the optimization variables - subset_forward = forward_i.intersection(links_p_nom.index) - subset_backward = backward_i.intersection(links_p_nom.index) + subset_forward = forward_i.intersection(links_p_nom.indexes["Link-ext"]) + subset_backward = backward_i.intersection(links_p_nom.indexes["Link-ext"]) # ensure we have a matching number of forward and backward links if len(subset_forward) != len(subset_backward): @@ -967,13 +967,10 @@ def get_backward_i(forward_i): # define the lefthand side of the constrain p_nom (forward) - p_nom (backward) = 0 # this ensures that the forward links always have the same maximum nominal power as their backward counterpart - lhs = linexpr( - (1, get_var(n, "Link", "p_nom")[backward_i].to_numpy()), - (-1, get_var(n, "Link", "p_nom")[forward_i].to_numpy()), - ) + lhs = links_p_nom.loc[backward_i] - links_p_nom.loc[forward_i] # add the constraint to the PySPA model - define_constraints(n, lhs, "=", 0, "Link-bidirectional_sync") + n.model.add_constraints(lhs == 0, name="Link-bidirectional_sync") def extra_functionality(n, snapshots): @@ -1102,11 +1099,17 @@ def solve_network(n, config, solving, **kwargs): from _helpers import mock_snakemake snakemake = mock_snakemake( - "solve_network", + "solve_sector_network", simpl="", clusters="4", ll="c1", opts="Co2L-4H", + planning_horizons="2030", + discountrate="0.071", + demand="AB", + sopts="144H", + h2export="120", + configfile="config.tutorial.yaml", ) configure_logging(snakemake) From 5d3e89a483e0e095955d12ab63af82ae819b00f8 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:55:05 +0100 Subject: [PATCH 58/68] Fix copy-minimal for windows (#1301) * Add abs path for overrides with copy-minimal * Update release note --- Snakefile | 9 +++++---- doc/release_notes.rst | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Snakefile b/Snakefile index 6653014a3..814a5d4b1 100644 --- a/Snakefile +++ b/Snakefile @@ -16,6 +16,7 @@ from _helpers import ( get_last_commit_message, check_config_version, copy_default_files, + BASE_DIR, ) from build_demand_profiles import get_load_paths_gegis from retrieve_databundle_light import datafiles_retrivedatabundle @@ -809,7 +810,7 @@ if config["monte_carlo"]["options"].get("add_to_snakefile", False) == False: solving=config["solving"], augmented_line_connection=config["augmented_line_connection"], input: - overrides="data/override_component_attrs", + overrides=BASE_DIR + "/data/override_component_attrs", network="networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: "results/" + RDIR + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", @@ -876,7 +877,7 @@ if config["monte_carlo"]["options"].get("add_to_snakefile", False) == True: solving=config["solving"], augmented_line_connection=config["augmented_line_connection"], input: - overrides="data/override_component_attrs", + overrides=BASE_DIR + "/data/override_component_attrs", network="networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{unc}.nc", @@ -1620,7 +1621,7 @@ if config["foresight"] == "overnight": solving=config["solving"], augmented_line_connection=config["augmented_line_connection"], input: - overrides="data/override_component_attrs", + overrides=BASE_DIR + "/data/override_component_attrs", # network=RESDIR # + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc", network=RESDIR @@ -2086,7 +2087,7 @@ if config["foresight"] == "myopic": "co2_sequestration_potential", 200 ), input: - overrides="data/override_component_attrs", + overrides=BASE_DIR + "/data/override_component_attrs", network=RESDIR + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc", costs=CDIR + "costs_{planning_horizons}.csv", diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4b6bce8ef..4843b1dfd 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -29,7 +29,7 @@ This part of documentation collects descriptive release notes to capture the mai * Fix lossy bidirectional links, especially H2 pipelines, which would sometimes gain H2 instead of losing it. `PR #1192 `__ -* Fix the need for administrative rights on Windows by changing all shadow directory settings for Windows in the Snakefile `PR #1295 `__ +* Fix the need for administrative rights on Windows by changing all shadow directory settings for Windows in the Snakefile `PR #1295 `__ and `PR #1301 `__ PyPSA-Earth 0.6.0 ================= From c44413cf4a568a1682c55dab6ac1658184281db8 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Thu, 16 Jan 2025 18:01:05 +0100 Subject: [PATCH 59/68] Update stable version in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c1f9e0ef3..d9325b303 100644 --- a/README.md +++ b/README.md @@ -121,10 +121,10 @@ There are multiple ways to get involved and learn more about our work: ## Running the model in previous versions -The model can be run in previous versions by checking out the respective tag. For instance, to run the model in version 0.4.1, which is the last version before the repo `pypsa-earth-sec` was merged, the following command can be used: +The model can be run in previous versions by checking out the respective tag. For instance, to run the model in version 0.6.0, which is the last version before the repo `pypsa-earth-sec` was merged, the following command can be used: ```bash -git checkout v0.4.1 +git checkout v0.6.0 ``` After checking out the tag, the model can be run as usual. Please make sure to install the required packages for the respective version. From e29386dfe1ce10da986d341f313797fd27c3cd08 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 22:18:11 +0100 Subject: [PATCH 60/68] [github-actions.ci] Update pinned envs (#1302) * Update pinned environment files for all platforms * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: davide-f <67809479+davide-f@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- envs/linux-pinned.yaml | 89 +++++++++++++++++++------------------- envs/macos-pinned.yaml | 92 ++++++++++++++++++++-------------------- envs/windows-pinned.yaml | 80 +++++++++++++++++----------------- 3 files changed, 130 insertions(+), 131 deletions(-) diff --git a/envs/linux-pinned.yaml b/envs/linux-pinned.yaml index d35941c07..f148d16fa 100644 --- a/envs/linux-pinned.yaml +++ b/envs/linux-pinned.yaml @@ -17,7 +17,7 @@ dependencies: - alsa-lib=1.2.13 - ampl-asl=1.0.0 - amply=0.1.6 -- anyio=4.7.0 +- anyio=4.8.0 - appdirs=1.4.4 - argon2-cffi=23.1.0 - argon2-cffi-bindings=21.2.0 @@ -37,9 +37,9 @@ dependencies: - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 - aws-c-s3=0.7.7 -- aws-c-sdkutils=0.2.1 +- aws-c-sdkutils=0.2.2 - aws-checksums=0.2.2 -- aws-crt-cpp=0.29.7 +- aws-crt-cpp=0.29.8 - aws-sdk-cpp=1.11.458 - azure-core-cpp=1.14.0 - azure-identity-cpp=1.10.0 @@ -49,6 +49,7 @@ dependencies: - babel=2.16.0 - beautifulsoup4=4.12.3 - bleach=6.2.0 +- bleach-with-css=6.2.0 - blosc=1.21.6 - bokeh=3.6.2 - bottleneck=1.4.2 @@ -70,13 +71,13 @@ dependencies: - certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 -- cfitsio=4.4.1 +- cfitsio=4.5.0 - cftime=1.6.4 -- charset-normalizer=3.4.0 +- charset-normalizer=3.4.1 - click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 -- cloudpickle=3.1.0 +- cloudpickle=3.1.1 - coin-or-cbc=2.10.12 - coin-or-cgl=0.60.9 - coin-or-clp=1.17.10 @@ -99,7 +100,7 @@ dependencies: - dask=2024.12.1 - dask-core=2024.12.1 - dask-expr=1.1.21 -- datapi=0.1.1 +- datapi=0.1.2 - datashader=0.16.3 - datrie=0.8.2 - dbus=1.13.6 @@ -122,7 +123,7 @@ dependencies: - filelock=3.16.1 - fiona=1.10.1 - fmt=11.0.2 -- folium=0.19.2 +- folium=0.19.4 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -152,8 +153,8 @@ dependencies: - gettext-tools=0.22.5 - gflags=2.2.2 - giflib=5.2.2 -- gitdb=4.0.11 -- gitpython=3.1.43 +- gitdb=4.0.12 +- gitpython=3.1.44 - glib=2.82.2 - glib-tools=2.82.2 - glog=0.7.1 @@ -168,9 +169,10 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=10.1.0 +- harfbuzz=10.2.0 - hdf4=4.2.15 - hdf5=1.14.3 +- highspy=1.9.0 - holoviews=1.20.0 - hpack=4.0.0 - httpcore=1.0.7 @@ -179,11 +181,11 @@ dependencies: - hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 -- identify=2.6.3 +- identify=2.6.5 - idna=3.10 - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 -- importlib_resources=6.4.5 +- importlib_resources=6.5.2 - inflate64=1.0.1 - iniconfig=2.0.0 - ipopt=3.14.17 @@ -208,7 +210,7 @@ dependencies: - jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 -- kealib=1.6.0 +- kealib=1.6.1 - keyutils=1.6.1 - kiwisolver=1.4.7 - krb5=1.21.3 @@ -231,14 +233,14 @@ dependencies: - libbrotlienc=1.1.0 - libcap=2.71 - libcblas=3.9.0 -- libclang-cpp19.1=19.1.6 -- libclang13=19.1.6 +- libclang-cpp19.1=19.1.7 +- libclang13=19.1.7 - libcrc32c=1.1.2 - libcups=2.3.3 - libcurl=8.11.1 - libdeflate=1.23 - libdrm=2.4.124 -- libedit=3.1.20191231 +- libedit=3.1.20240808 - libegl=1.7.0 - libev=4.33 - libevent=2.1.12 @@ -273,8 +275,8 @@ dependencies: - libglvnd=1.7.0 - libglx=1.7.0 - libgomp=14.2.0 -- libgoogle-cloud=2.32.0 -- libgoogle-cloud-storage=2.32.0 +- libgoogle-cloud=2.33.0 +- libgoogle-cloud-storage=2.33.0 - libgpg-error=1.51 - libgrpc=1.67.1 - libhwloc=2.11.2 @@ -284,7 +286,7 @@ dependencies: - liblapack=3.9.0 - liblapacke=3.9.0 - libllvm14=14.0.6 -- libllvm19=19.1.6 +- libllvm19=19.1.7 - liblzma=5.6.3 - liblzma-devel=5.6.3 - libmamba=2.0.5 @@ -297,9 +299,9 @@ dependencies: - libopus=1.3.1 - libparquet=18.1.0 - libpciaccess=0.18 -- libpng=1.6.44 +- libpng=1.6.45 - libpq=17.2 -- libprotobuf=5.28.2 +- libprotobuf=5.28.3 - libre2-11=2024.07.02 - librsvg=2.58.4 - librttopo=1.1.0 @@ -307,14 +309,13 @@ dependencies: - libsndfile=1.2.2 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.1.0 - libspatialite=5.1.0 - libspral=2024.05.08 - libsqlite=3.47.2 - libssh2=1.11.1 - libstdcxx=14.2.0 - libstdcxx-ng=14.2.0 -- libsystemd0=256.9 +- libsystemd0=257.2 - libthrift=0.21.0 - libtiff=4.7.0 - libutf8proc=2.9.0 @@ -350,7 +351,7 @@ dependencies: - mercantile=1.2.1 - metis=5.1.0 - minizip=4.0.7 -- mistune=3.0.2 +- mistune=3.1.0 - mpfr=4.2.1 - mpg123=1.32.9 - msgpack-python=1.1.0 @@ -363,7 +364,7 @@ dependencies: - mysql-common=9.0.1 - mysql-libs=9.0.1 - nbclient=0.10.2 -- nbconvert-core=7.16.4 +- nbconvert-core=7.16.5 - nbformat=5.10.4 - ncurses=6.5 - nest-asyncio=1.6.0 @@ -389,7 +390,7 @@ dependencies: - pandas=2.2.2 - pandocfilters=1.5.0 - panel=1.5.5 -- pango=1.54.0 +- pango=1.56.0 - param=2.2.0 - parso=0.8.4 - partd=1.4.2 @@ -397,7 +398,7 @@ dependencies: - pcre2=10.44 - pexpect=4.9.0 - pickleshare=0.7.5 -- pillow=11.0.0 +- pillow=11.1.0 - pip=24.3.1 - pixman=0.44.2 - pkgutil-resolve-name=1.3.10 @@ -409,14 +410,14 @@ dependencies: - poppler=24.12.0 - poppler-data=0.4.12 - postgresql=17.2 -- powerplantmatching=0.6.0 +- powerplantmatching=0.6.1 - pre-commit=4.0.1 - progressbar2=4.5.0 - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 -- protobuf=5.28.2 -- psutil=6.1.0 +- protobuf=5.28.3 +- psutil=6.1.1 - pthread-stubs=0.4 - ptyprocess=0.7.0 - pulp=2.7.0 @@ -432,10 +433,10 @@ dependencies: - pycryptodomex=3.21.0 - pyct=0.5.0 - pydoe2=1.3.0 -- pygments=2.18.0 +- pygments=2.19.1 - pyogrio=0.10.0 - pyomo=6.6.1 -- pyparsing=3.2.0 +- pyparsing=3.2.1 - pyppmd=1.1.0 - pyproj=3.7.0 - pypsa=0.28.0 @@ -454,7 +455,7 @@ dependencies: - python-utils=3.9.1 - python_abi=3.10 - pytz=2024.2 -- pyviz_comms=3.0.3 +- pyviz_comms=3.0.4 - pyyaml=6.0.2 - pyzmq=26.2.0 - pyzstd=0.16.2 @@ -475,15 +476,15 @@ dependencies: - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 - s2n=1.5.10 -- scikit-learn=1.6.0 +- scikit-learn=1.6.1 - scip=9.2.0 -- scipy=1.14.1 +- scipy=1.15.1 - seaborn=0.13.2 - seaborn-base=0.13.2 - send2trash=1.8.3 -- setuptools=75.6.0 +- setuptools=75.8.0 - shapely=2.0.6 -- simdjson=3.11.3 +- simdjson=3.11.6 - sip=6.7.12 - six=1.17.0 - smart_open=7.1.0 @@ -494,7 +495,7 @@ dependencies: - snuggs=1.4.7 - sortedcontainers=2.4.0 - soupsieve=2.5 -- spdlog=1.14.1 +- spdlog=1.15.0 - sqlite=3.47.2 - stack_data=0.6.3 - statsmodels=0.14.4 @@ -525,19 +526,19 @@ dependencies: - tzdata=2024b - uc-micro-py=1.0.3 - ukkonen=1.0.1 -- unicodedata2=15.1.0 +- unicodedata2=16.0.0 - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 - urllib3=2.3.0 - validators=0.34.0 -- virtualenv=20.28.0 +- virtualenv=20.29.0 - wcwidth=0.2.13 - webcolors=24.11.1 - webencodings=0.5.1 - websocket-client=1.8.0 - wheel=0.45.1 -- wrapt=1.17.0 +- wrapt=1.17.2 - xarray=2023.11.0 - xcb-util=0.4.1 - xcb-util-image=0.4.0 @@ -569,13 +570,11 @@ dependencies: - zipfile-deflate64=0.2.0 - zipp=3.21.0 - zlib=1.3.1 -- zlib-ng=2.2.2 +- zlib-ng=2.2.3 - zstandard=0.23.0 - zstd=1.5.6 - pip: - chaospy==4.3.17 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.9.0 - - tsam==2.3.6 prefix: /usr/share/miniconda/envs/pypsa-earth diff --git a/envs/macos-pinned.yaml b/envs/macos-pinned.yaml index c6d1be082..ba162351c 100644 --- a/envs/macos-pinned.yaml +++ b/envs/macos-pinned.yaml @@ -14,7 +14,7 @@ dependencies: - affine=2.4.0 - ampl-asl=1.0.0 - amply=0.1.6 -- anyio=4.7.0 +- anyio=4.8.0 - appdirs=1.4.4 - appnope=0.1.4 - argon2-cffi=23.1.0 @@ -34,9 +34,9 @@ dependencies: - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 - aws-c-s3=0.7.7 -- aws-c-sdkutils=0.2.1 +- aws-c-sdkutils=0.2.2 - aws-checksums=0.2.2 -- aws-crt-cpp=0.29.7 +- aws-crt-cpp=0.29.8 - aws-sdk-cpp=1.11.458 - azure-core-cpp=1.14.0 - azure-identity-cpp=1.10.0 @@ -46,6 +46,7 @@ dependencies: - babel=2.16.0 - beautifulsoup4=4.12.3 - bleach=6.2.0 +- bleach-with-css=6.2.0 - blosc=1.21.6 - bokeh=3.6.2 - bottleneck=1.4.2 @@ -67,13 +68,13 @@ dependencies: - certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 -- cfitsio=4.4.1 +- cfitsio=4.5.0 - cftime=1.6.4 -- charset-normalizer=3.4.0 +- charset-normalizer=3.4.1 - click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 -- cloudpickle=3.1.0 +- cloudpickle=3.1.1 - coin-or-cbc=2.10.12 - coin-or-cgl=0.60.9 - coin-or-clp=1.17.10 @@ -96,7 +97,7 @@ dependencies: - dask=2024.12.1 - dask-core=2024.12.1 - dask-expr=1.1.21 -- datapi=0.1.1 +- datapi=0.1.2 - datashader=0.16.3 - datrie=0.8.2 - debugpy=1.8.11 @@ -117,7 +118,7 @@ dependencies: - filelock=3.16.1 - fiona=1.10.1 - fmt=11.0.2 -- folium=0.19.2 +- folium=0.19.4 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -145,8 +146,8 @@ dependencies: - geoviews-core=1.14.0 - gflags=2.2.2 - giflib=5.2.2 -- gitdb=4.0.11 -- gitpython=3.1.43 +- gitdb=4.0.12 +- gitpython=3.1.44 - glog=0.7.1 - glpk=5.0 - gmp=6.3.0 @@ -157,9 +158,10 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=10.1.0 +- harfbuzz=10.2.0 - hdf4=4.2.15 - hdf5=1.14.3 +- highspy=1.9.0 - holoviews=1.20.0 - hpack=4.0.0 - httpcore=1.0.7 @@ -168,11 +170,11 @@ dependencies: - hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 -- identify=2.6.3 +- identify=2.6.5 - idna=3.10 - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 -- importlib_resources=6.4.5 +- importlib_resources=6.5.2 - inflate64=1.0.1 - iniconfig=2.0.0 - ipopt=3.14.17 @@ -197,7 +199,7 @@ dependencies: - jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 -- kealib=1.6.0 +- kealib=1.6.1 - kiwisolver=1.4.7 - krb5=1.21.3 - lcms2=2.16 @@ -216,9 +218,9 @@ dependencies: - libcblas=3.9.0 - libcrc32c=1.1.2 - libcurl=8.11.1 -- libcxx=19.1.6 +- libcxx=19.1.7 - libdeflate=1.23 -- libedit=3.1.20191231 +- libedit=3.1.20240808 - libev=4.33 - libevent=2.1.12 - libexpat=2.6.4 @@ -241,8 +243,8 @@ dependencies: - libgfortran=5.0.0 - libgfortran5=13.2.0 - libglib=2.82.2 -- libgoogle-cloud=2.32.0 -- libgoogle-cloud-storage=2.32.0 +- libgoogle-cloud=2.33.0 +- libgoogle-cloud-storage=2.33.0 - libgrpc=1.67.1 - libhwloc=2.11.2 - libiconv=1.17 @@ -259,16 +261,15 @@ dependencies: - libntlm=1.8 - libopenblas=0.3.28 - libparquet=18.1.0 -- libpng=1.6.44 +- libpng=1.6.45 - libpq=17.2 -- libprotobuf=5.28.2 +- libprotobuf=5.28.3 - libre2-11=2024.07.02 - librsvg=2.58.4 - librttopo=1.1.0 -- libscotch=7.0.5 +- libscotch=7.0.6 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.1.0 - libspatialite=5.1.0 - libsqlite=3.47.2 - libssh2=1.11.1 @@ -283,7 +284,7 @@ dependencies: - libzlib=1.3.1 - linkify-it-py=2.0.3 - linopy=0.3.11 -- llvm-openmp=19.1.6 +- llvm-openmp=19.1.7 - llvmlite=0.43.0 - locket=1.0.0 - lxml=5.3.0 @@ -304,7 +305,7 @@ dependencies: - mercantile=1.2.1 - metis=5.1.0 - minizip=4.0.7 -- mistune=3.0.2 +- mistune=3.1.0 - mpfr=4.2.1 - msgpack-python=1.1.0 - multipledispatch=0.6.0 @@ -314,7 +315,7 @@ dependencies: - mumps-seq=5.7.3 - munkres=1.1.4 - nbclient=0.10.2 -- nbconvert-core=7.16.4 +- nbconvert-core=7.16.5 - nbformat=5.10.4 - ncurses=6.5 - nest-asyncio=1.6.0 @@ -339,7 +340,7 @@ dependencies: - pandas=2.2.2 - pandocfilters=1.5.0 - panel=1.5.5 -- pango=1.54.0 +- pango=1.56.0 - param=2.2.0 - parso=0.8.4 - partd=1.4.2 @@ -347,7 +348,7 @@ dependencies: - pcre2=10.44 - pexpect=4.9.0 - pickleshare=0.7.5 -- pillow=11.0.0 +- pillow=11.1.0 - pip=24.3.1 - pixman=0.44.2 - pkgutil-resolve-name=1.3.10 @@ -359,14 +360,14 @@ dependencies: - poppler=24.12.0 - poppler-data=0.4.12 - postgresql=17.2 -- powerplantmatching=0.6.0 +- powerplantmatching=0.6.1 - pre-commit=4.0.1 - progressbar2=4.5.0 - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 -- protobuf=5.28.2 -- psutil=6.1.0 +- protobuf=5.28.3 +- psutil=6.1.1 - pthread-stubs=0.4 - ptyprocess=0.7.0 - pulp=2.7.0 @@ -381,12 +382,12 @@ dependencies: - pycryptodomex=3.21.0 - pyct=0.5.0 - pydoe2=1.3.0 -- pygments=2.18.0 -- pyobjc-core=10.3.2 -- pyobjc-framework-cocoa=10.3.2 +- pygments=2.19.1 +- pyobjc-core=11.0 +- pyobjc-framework-cocoa=11.0 - pyogrio=0.10.0 - pyomo=6.6.1 -- pyparsing=3.2.0 +- pyparsing=3.2.1 - pyppmd=1.1.0 - pyproj=3.7.0 - pypsa=0.28.0 @@ -403,7 +404,7 @@ dependencies: - python-utils=3.9.1 - python_abi=3.10 - pytz=2024.2 -- pyviz_comms=3.0.3 +- pyviz_comms=3.0.4 - pyyaml=6.0.2 - pyzmq=26.2.0 - pyzstd=0.16.2 @@ -422,15 +423,15 @@ dependencies: - rpds-py=0.22.3 - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 -- scikit-learn=1.6.0 +- scikit-learn=1.6.1 - scip=9.2.0 -- scipy=1.14.1 +- scipy=1.15.1 - seaborn=0.13.2 - seaborn-base=0.13.2 - send2trash=1.8.3 -- setuptools=75.6.0 +- setuptools=75.8.0 - shapely=2.0.6 -- simdjson=3.11.3 +- simdjson=3.11.6 - six=1.17.0 - smart_open=7.1.0 - smmap=5.0.0 @@ -440,7 +441,7 @@ dependencies: - snuggs=1.4.7 - sortedcontainers=2.4.0 - soupsieve=2.5 -- spdlog=1.14.1 +- spdlog=1.15.0 - sqlite=3.47.2 - stack_data=0.6.3 - statsmodels=0.14.4 @@ -470,22 +471,23 @@ dependencies: - tzdata=2024b - uc-micro-py=1.0.3 - ukkonen=1.0.1 -- unicodedata2=15.1.0 +- unicodedata2=16.0.0 - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 - urllib3=2.3.0 - validators=0.34.0 -- virtualenv=20.28.0 +- virtualenv=20.29.0 - wcwidth=0.2.13 - webcolors=24.11.1 - webencodings=0.5.1 - websocket-client=1.8.0 - wheel=0.45.1 -- wrapt=1.17.0 +- wrapt=1.17.2 - xarray=2023.11.0 - xerces-c=3.2.5 - xlrd=2.0.1 +- xorg-libxau=1.0.12 - xorg-libxdmcp=1.1.5 - xyzservices=2024.9.0 - yaml=0.2.5 @@ -496,13 +498,11 @@ dependencies: - zipfile-deflate64=0.2.0 - zipp=3.21.0 - zlib=1.3.1 -- zlib-ng=2.2.2 +- zlib-ng=2.2.3 - zstandard=0.23.0 - zstd=1.5.6 - pip: - chaospy==4.3.17 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.9.0 - - tsam==2.3.6 prefix: /Users/runner/miniconda3/envs/pypsa-earth diff --git a/envs/windows-pinned.yaml b/envs/windows-pinned.yaml index c43b70e60..deaf33f2b 100644 --- a/envs/windows-pinned.yaml +++ b/envs/windows-pinned.yaml @@ -16,7 +16,7 @@ dependencies: - affine=2.4.0 - ampl-asl=1.0.0 - amply=0.1.6 -- anyio=4.7.0 +- anyio=4.8.0 - appdirs=1.4.4 - argon2-cffi=23.1.0 - argon2-cffi-bindings=21.2.0 @@ -34,9 +34,9 @@ dependencies: - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 - aws-c-s3=0.7.7 -- aws-c-sdkutils=0.2.1 +- aws-c-sdkutils=0.2.2 - aws-checksums=0.2.2 -- aws-crt-cpp=0.29.7 +- aws-crt-cpp=0.29.8 - aws-sdk-cpp=1.11.458 - azure-core-cpp=1.14.0 - azure-identity-cpp=1.10.0 @@ -45,6 +45,7 @@ dependencies: - babel=2.16.0 - beautifulsoup4=4.12.3 - bleach=6.2.0 +- bleach-with-css=6.2.0 - blosc=1.21.6 - bokeh=3.6.2 - bottleneck=1.4.2 @@ -66,13 +67,13 @@ dependencies: - certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 -- cfitsio=4.4.1 +- cfitsio=4.5.0 - cftime=1.6.4 -- charset-normalizer=3.4.0 +- charset-normalizer=3.4.1 - click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 -- cloudpickle=3.1.0 +- cloudpickle=3.1.1 - colorama=0.4.6 - colorcet=3.1.0 - comm=0.2.2 @@ -89,7 +90,7 @@ dependencies: - dask=2024.12.1 - dask-core=2024.12.1 - dask-expr=1.1.21 -- datapi=0.1.1 +- datapi=0.1.2 - datashader=0.16.3 - datrie=0.8.2 - debugpy=1.8.11 @@ -110,7 +111,7 @@ dependencies: - filelock=3.16.1 - fiona=1.10.1 - fmt=11.0.2 -- folium=0.19.2 +- folium=0.19.4 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -136,8 +137,8 @@ dependencies: - geoviews=1.14.0 - geoviews-core=1.14.0 - getopt-win32=0.1 -- gitdb=4.0.11 -- gitpython=3.1.43 +- gitdb=4.0.12 +- gitpython=3.1.44 - glib=2.82.2 - glib-tools=2.82.2 - glpk=5.0 @@ -150,9 +151,10 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=10.1.0 +- harfbuzz=10.2.0 - hdf4=4.2.15 - hdf5=1.14.3 +- highspy=1.9.0 - holoviews=1.20.0 - hpack=4.0.0 - httpcore=1.0.7 @@ -161,11 +163,11 @@ dependencies: - hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 -- identify=2.6.3 +- identify=2.6.5 - idna=3.10 - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 -- importlib_resources=6.4.5 +- importlib_resources=6.5.2 - inflate64=1.0.1 - iniconfig=2.0.0 - ipopt=3.14.17 @@ -189,7 +191,7 @@ dependencies: - jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 -- kealib=1.6.0 +- kealib=1.6.1 - kiwisolver=1.4.7 - krb5=1.21.3 - lcms2=2.16 @@ -207,7 +209,7 @@ dependencies: - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 -- libclang13=19.1.6 +- libclang13=19.1.7 - libcrc32c=1.1.2 - libcurl=8.11.1 - libdeflate=1.23 @@ -233,8 +235,8 @@ dependencies: - libgdal-xls=3.9.3 - libglib=2.82.2 - libgomp=14.2.0 -- libgoogle-cloud=2.32.0 -- libgoogle-cloud-storage=2.32.0 +- libgoogle-cloud=2.33.0 +- libgoogle-cloud-storage=2.33.0 - libgrpc=1.67.1 - libhwloc=2.11.2 - libiconv=1.17 @@ -249,14 +251,13 @@ dependencies: - libogg=1.3.5 - libopenblas=0.3.28 - libparquet=18.1.0 -- libpng=1.6.44 +- libpng=1.6.45 - libpq=17.2 -- libprotobuf=5.28.2 +- libprotobuf=5.28.3 - libre2-11=2024.07.02 - librttopo=1.1.0 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.1.0 - libspatialite=5.1.0 - libsqlite=3.47.2 - libssh2=1.11.1 @@ -293,7 +294,7 @@ dependencies: - memory_profiler=0.61.0 - mercantile=1.2.1 - minizip=4.0.7 -- mistune=3.0.2 +- mistune=3.1.0 - mpfr=4.2.1 - msgpack-python=1.1.0 - multipledispatch=0.6.0 @@ -302,7 +303,7 @@ dependencies: - mumps-seq=5.7.3 - munkres=1.1.4 - nbclient=0.10.2 -- nbconvert-core=7.16.4 +- nbconvert-core=7.16.5 - nbformat=5.10.4 - nest-asyncio=1.6.0 - netcdf4=1.7.2 @@ -325,14 +326,14 @@ dependencies: - pandas=2.2.2 - pandocfilters=1.5.0 - panel=1.5.5 -- pango=1.54.0 +- pango=1.56.0 - param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 - pcre2=10.44 - pickleshare=0.7.5 -- pillow=11.0.0 +- pillow=11.1.0 - pip=24.3.1 - pixman=0.44.2 - pkgutil-resolve-name=1.3.10 @@ -343,14 +344,14 @@ dependencies: - poppler=24.12.0 - poppler-data=0.4.12 - postgresql=17.2 -- powerplantmatching=0.6.0 +- powerplantmatching=0.6.1 - pre-commit=4.0.1 - progressbar2=4.5.0 - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 -- protobuf=5.28.2 -- psutil=6.1.0 +- protobuf=5.28.3 +- psutil=6.1.1 - pthread-stubs=0.4 - pulp=2.7.0 - pure_eval=0.2.3 @@ -364,10 +365,10 @@ dependencies: - pycryptodomex=3.21.0 - pyct=0.5.0 - pydoe2=1.3.0 -- pygments=2.18.0 +- pygments=2.19.1 - pyogrio=0.10.0 - pyomo=6.6.1 -- pyparsing=3.2.0 +- pyparsing=3.2.1 - pyppmd=1.1.0 - pyproj=3.7.0 - pypsa=0.28.0 @@ -387,7 +388,7 @@ dependencies: - python-utils=3.9.1 - python_abi=3.10 - pytz=2024.2 -- pyviz_comms=3.0.3 +- pyviz_comms=3.0.4 - pywin32=307 - pywinpty=2.0.14 - pyyaml=6.0.2 @@ -408,15 +409,15 @@ dependencies: - rpds-py=0.22.3 - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 -- scikit-learn=1.6.0 +- scikit-learn=1.6.1 - scip=9.2.0 -- scipy=1.14.1 +- scipy=1.15.1 - seaborn=0.13.2 - seaborn-base=0.13.2 - send2trash=1.8.3 -- setuptools=75.6.0 +- setuptools=75.8.0 - shapely=2.0.6 -- simdjson=3.11.3 +- simdjson=3.11.6 - sip=6.7.12 - six=1.17.0 - smart_open=7.1.0 @@ -427,7 +428,7 @@ dependencies: - snuggs=1.4.7 - sortedcontainers=2.4.0 - soupsieve=2.5 -- spdlog=1.14.1 +- spdlog=1.15.0 - sqlite=3.47.2 - stack_data=0.6.3 - statsmodels=0.14.4 @@ -458,7 +459,7 @@ dependencies: - uc-micro-py=1.0.3 - ucrt=10.0.22621.0 - ukkonen=1.0.1 -- unicodedata2=15.1.0 +- unicodedata2=16.0.0 - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 @@ -466,7 +467,7 @@ dependencies: - validators=0.34.0 - vc=14.3 - vc14_runtime=14.42.34433 -- virtualenv=20.28.0 +- virtualenv=20.29.0 - vs2015_runtime=14.42.34433 - wcwidth=0.2.13 - webcolors=24.11.1 @@ -475,7 +476,7 @@ dependencies: - wheel=0.45.1 - win_inet_pton=1.1.0 - winpty=0.4.3 -- wrapt=1.17.0 +- wrapt=1.17.2 - xarray=2023.11.0 - xerces-c=3.2.5 - xlrd=2.0.1 @@ -496,13 +497,12 @@ dependencies: - zipfile-deflate64=0.2.0 - zipp=3.21.0 - zlib=1.3.1 -- zlib-ng=2.2.2 +- zlib-ng=2.2.3 - zstandard=0.23.0 - zstd=1.5.6 - pip: - chaospy==4.3.17 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.9.0 - polars==1.17.1 prefix: C:\Miniconda\envs\pypsa-earth From 5692063bce83dbe34435b1987756692108bab394 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Tue, 21 Jan 2025 00:58:10 +0100 Subject: [PATCH 61/68] Switch-off fetching isolated networks as a temporary fix (#1313) --- config.default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.default.yaml b/config.default.yaml index 56e03a56e..224f5f237 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -84,7 +84,7 @@ cluster_options: remove_stubs_across_borders: true p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold - s_threshold_fetch_isolated: 0.05 # [-] a share of the national load for merging an isolated network into a backbone network + s_threshold_fetch_isolated: false # [-] a share of the national load for merging an isolated network into a backbone network cluster_network: algorithm: kmeans feature: solar+onwind-time From 67df046be7ef36a0bdfb445e54f12093f24ed2ac Mon Sep 17 00:00:00 2001 From: Tobias Augspurger Date: Tue, 21 Jan 2025 11:27:44 +0100 Subject: [PATCH 62/68] add auto generated citation.cff --- CITATION.cff | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..8e8ab83e2 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,33 @@ +title: PyPSA-Earth. A new global open energy system optimization model demonstrated in Africa +abstract: This repository contains the source code of the paper PyPSA-Earth. A new global open energy system optimization model demonstrated in Africa by Maximilian Parzen, Hazem Abdel-Khalek, Ekaterina Fedotova, Matin Mahmood, Martha Maria Frysztacki, Johannes Hampp, Lukas Franken, Leon Schumm, Fabian Neumann, Davide Poli, Aristides Kiprakis and Davide Fioriti, from Applied Energy +doi: 10.1016/j.apenergy.2023.121096 +repository-code: https://github.com/pypsa-meets-earth/pypsa-earth +version: 1.0.0 +date-released: 2023-04-18 +message: If you use this software in your work, please cite it using the following metadata. +authors: + - given-names: Maximilian + family-names: Parzen + - given-names: Hazem + family-names: Abdel-Khalek + - given-names: Ekaterina + family-names: Fedotova + - given-names: Matin + family-names: Mahmood + - given-names: Martha Maria + family-names: Frysztacki + - given-names: Johannes + family-names: Hampp + - given-names: Lukas + family-names: Franken + - given-names: Leon + family-names: Schumm + - given-names: Fabian + family-names: Neumann + - given-names: Davide + family-names: Poli + - given-names: Aristides + family-names: Kiprakis + - given-names: Davide + family-names: Fioriti +cff-version: 1.2.0 From 9a2797ec93aa93f31ec32ffb05de59753851a622 Mon Sep 17 00:00:00 2001 From: Tobias Augspurger Date: Tue, 21 Jan 2025 11:35:04 +0100 Subject: [PATCH 63/68] add DOI badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d9325b303..72951d12a 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ by [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pypsa-meets-earth/pypsa-earth/main.svg)](https://results.pre-commit.ci/latest/github/pypsa-meets-earth/pypsa-earth/main) [![Discord](https://img.shields.io/discord/911692131440148490?logo=discord)](https://discord.gg/AnuJBk23FU) [![Google Drive](https://img.shields.io/badge/Google%20Drive-4285F4?style=flat&logo=googledrive&logoColor=white)](https://drive.google.com/drive/folders/13Z8Y9zgsh5IZaDNkkRyo1wkoMgbdUxT5?usp=sharing) +[![DOI](https://img.shields.io/badge/10.1016/j.apenergy.2023.121096)](https://doi.org/10.1016/j.apenergy.2023.121096) + **PyPSA-Earth: A Global Sector-Coupled Open-Source Multi-Energy System Model** From 27076dbcec3a9039dfae3ca7749436fe848a0988 Mon Sep 17 00:00:00 2001 From: Tobias Augspurger Date: Tue, 21 Jan 2025 11:58:23 +0100 Subject: [PATCH 64/68] Add ORCID --- CITATION.cff | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CITATION.cff b/CITATION.cff index 8e8ab83e2..8c93b0446 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,26 +8,33 @@ message: If you use this software in your work, please cite it using the followi authors: - given-names: Maximilian family-names: Parzen + orcid: https://orcid.org/0000-0002-4390-0063 - given-names: Hazem family-names: Abdel-Khalek - given-names: Ekaterina family-names: Fedotova + orcid: https://orcid.org/0000-0002-5590-9591 - given-names: Matin family-names: Mahmood - given-names: Martha Maria family-names: Frysztacki + orcid: https://orcid.org/0000-0002-0788-1328 - given-names: Johannes family-names: Hampp + orcid: https://orcid.org/0000-0002-1776-116X - given-names: Lukas family-names: Franken - given-names: Leon family-names: Schumm - given-names: Fabian family-names: Neumann + orcid: https://orcid.org/0000-0002-6604-5450 - given-names: Davide family-names: Poli + orcid: https://orcid.org/0000-0002-5045-9034 - given-names: Aristides family-names: Kiprakis - given-names: Davide family-names: Fioriti + orcid: https://orcid.org/0000-0001-5491-7912 cff-version: 1.2.0 From d1fc82bb9ca98bc9d7f40f4485cdbb6157d9861c Mon Sep 17 00:00:00 2001 From: Tobias Augspurger Date: Tue, 21 Jan 2025 12:26:43 +0100 Subject: [PATCH 65/68] fix bug in badge and add linter note to file --- CITATION.cff | 5 +++++ README.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 8c93b0446..54260fd89 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,3 +1,8 @@ + title: PyPSA-Earth. A new global open energy system optimization model demonstrated in Africa abstract: This repository contains the source code of the paper PyPSA-Earth. A new global open energy system optimization model demonstrated in Africa by Maximilian Parzen, Hazem Abdel-Khalek, Ekaterina Fedotova, Matin Mahmood, Martha Maria Frysztacki, Johannes Hampp, Lukas Franken, Leon Schumm, Fabian Neumann, Davide Poli, Aristides Kiprakis and Davide Fioriti, from Applied Energy doi: 10.1016/j.apenergy.2023.121096 diff --git a/README.md b/README.md index 72951d12a..0aac8b5db 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ by [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pypsa-meets-earth/pypsa-earth/main.svg)](https://results.pre-commit.ci/latest/github/pypsa-meets-earth/pypsa-earth/main) [![Discord](https://img.shields.io/discord/911692131440148490?logo=discord)](https://discord.gg/AnuJBk23FU) [![Google Drive](https://img.shields.io/badge/Google%20Drive-4285F4?style=flat&logo=googledrive&logoColor=white)](https://drive.google.com/drive/folders/13Z8Y9zgsh5IZaDNkkRyo1wkoMgbdUxT5?usp=sharing) -[![DOI](https://img.shields.io/badge/10.1016/j.apenergy.2023.121096)](https://doi.org/10.1016/j.apenergy.2023.121096) +[![DOI](https://img.shields.io/badge/DOI-10.1016%2Fj.apenergy.2023.121096-blue)](https://doi.org/10.1016/j.apenergy.2023.121096) **PyPSA-Earth: A Global Sector-Coupled Open-Source Multi-Energy System Model** From 1a6259eb238907ab7f1d62b3e255115192ad2085 Mon Sep 17 00:00:00 2001 From: Tobias Augspurger Date: Tue, 21 Jan 2025 13:01:39 +0100 Subject: [PATCH 66/68] Update CITATION.cff Add header for reuse linter --- CITATION.cff | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 54260fd89..9b29144b4 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,8 +1,6 @@ - title: PyPSA-Earth. A new global open energy system optimization model demonstrated in Africa abstract: This repository contains the source code of the paper PyPSA-Earth. A new global open energy system optimization model demonstrated in Africa by Maximilian Parzen, Hazem Abdel-Khalek, Ekaterina Fedotova, Matin Mahmood, Martha Maria Frysztacki, Johannes Hampp, Lukas Franken, Leon Schumm, Fabian Neumann, Davide Poli, Aristides Kiprakis and Davide Fioriti, from Applied Energy doi: 10.1016/j.apenergy.2023.121096 From 7d83f351c9a49c70efd3895e1e131992aa7ef3d0 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Tue, 21 Jan 2025 17:07:19 +0100 Subject: [PATCH 67/68] Align structure of the components with PyPSA consistency checks (#1315) * Remove supplementary data from lines dataframe * Update the clustering strategy * Improve dealing with the frequency field * Remove redundand use of the frequency tag * Add release note --- doc/release_notes.rst | 2 ++ scripts/base_network.py | 5 ++--- scripts/build_osm_network.py | 4 ---- scripts/cluster_network.py | 2 -- scripts/simplify_network.py | 2 -- 5 files changed, 4 insertions(+), 11 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b28b545b5..7d8be983c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -25,6 +25,8 @@ This part of documentation collects descriptive release notes to capture the mai **Minor Changes and bug-fixing** +* Align structure of the components with consistency checks in the updated PyPSA version `PR #1315 `__ + * Prevent computation of powerplantmatching if replace option is selected for custom_powerplants `PR #1281 `__ * Fix overlapping bus regions when alternative clustering is selected `PR #1287 `__ diff --git a/scripts/base_network.py b/scripts/base_network.py index e11ff83c6..2c4aa6afd 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -489,9 +489,8 @@ def base_network( transformers = _load_transformers_from_osm(inputs.osm_transformers, buses) converters = _load_converters_from_osm(inputs.osm_converters, buses) - lines_ac = lines[lines.tag_frequency.astype(float) != 0].copy() - lines_dc = lines[lines.tag_frequency.astype(float) == 0].copy() - + lines_ac = lines[~lines.dc].copy() + lines_dc = lines[lines.dc].copy() lines_ac = _set_electrical_parameters_lines(lines_config, voltages_config, lines_ac) lines_dc = _set_electrical_parameters_dc_lines( diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index de88be8e5..91a66ef5f 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -32,14 +32,10 @@ LINES_COLUMNS = [ "line_id", "circuits", - "tag_type", "voltage", "bus0", "bus1", "length", - "underground", - "under_construction", - "tag_frequency", "dc", "country", "geometry", diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 562b9cfa7..6d97216f6 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -736,8 +736,6 @@ def consense(x): "v_nom": "first", "lat": "mean", "lon": "mean", - "tag_substation": "first", - "tag_area": "first", "country": "first", }, ) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 502cf1b9d..9e3576388 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -1003,8 +1003,6 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()): "v_nom": "first", "lat": "mean", "lon": "mean", - "tag_substation": "first", - "tag_area": "first", "country": "first", }, ) From 52200519b7efa576848cd2acf2accc3bedc2abd8 Mon Sep 17 00:00:00 2001 From: Tobias Augspurger Date: Wed, 22 Jan 2025 09:20:53 +0100 Subject: [PATCH 68/68] Add new CITATION.cff to release_notes.rst --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b28b545b5..22b69a4e1 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -35,6 +35,8 @@ This part of documentation collects descriptive release notes to capture the mai * Fix the need for administrative rights on Windows by changing all shadow directory settings for Windows in the Snakefile `PR #1295 `__ and `PR #1301 `__ +* Add CITATION.cff to guide users on how cite PyPSA-Earth. Insert the DOI badge in the README linking to the very first PyPSA-Earth paper. `PR #1316 `__ + PyPSA-Earth 0.6.0 =================