From e4283bd5cf44fa3846fd31a08f4a0734a32d350c Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Wed, 19 Jul 2023 19:39:37 +0100 Subject: [PATCH 001/113] optimize set_substation_ids using DBSCAN --- scripts/build_osm_network.py | 83 +++++++++--------------------------- 1 file changed, 20 insertions(+), 63 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index b89180c94..16511cd37 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -16,6 +16,7 @@ from shapely.geometry import LineString, Point from shapely.ops import linemerge, split from tqdm import tqdm +from sklearn.cluster import DBSCAN logger = logging.getLogger(__name__) @@ -37,76 +38,32 @@ def line_endings_to_bus_conversion(lines): return lines -# tol in m -def set_substations_ids(buses, distance_crs, tol=2000): - """ - Function to set substations ids to buses, accounting for location - tolerance. - - The algorithm is as follows: - - 1. initialize all substation ids to -1 - 2. if the current substation has been already visited [substation_id < 0], then skip the calculation - 3. otherwise: - 1. identify the substations within the specified tolerance (tol) - 2. when all the substations in tolerance have substation_id < 0, then specify a new substation_id - 3. otherwise, if one of the substation in tolerance has a substation_id >= 0, then set that substation_id to all the others; - in case of multiple substations with substation_ids >= 0, the first value is picked for all +def set_substations_ids(buses, distance_crs, tol=5000): """ + Assigns station IDs to buses based on their proximity. - buses["station_id"] = -1 + Parameters: + - buses: GeoDataFrame object representing the buses data. + - distance_crs: Coordinate reference system (CRS) to convert the geometry to. + - tol: Tolerance distance in chosen CRS to define cluster proximity. - # create temporary series to execute distance calculations using m as reference distances - temp_bus_geom = buses.geometry.to_crs(distance_crs) + Returns: + - None. Modifies the 'station_id' column in the 'buses' GeoDataFrame. - # set tqdm options for substation ids - tqdm_kwargs_substation_ids = dict( - ascii=False, - unit=" buses", - total=buses.shape[0], - desc="Set substation ids ", - ) + Example: + set_substations_ids(buses_data, 'EPSG:3857', tol=5000) + """ - station_id = 0 - for i, row in tqdm(buses.iterrows(), **tqdm_kwargs_substation_ids): - if buses.loc[i, "station_id"] >= 0: - continue + # Convert the geometry to EPSG:3857 + tmp_geometry = buses.geometry.to_crs(distance_crs) - # get substations within tolerance - close_nodes = np.flatnonzero( - temp_bus_geom.distance(temp_bus_geom.loc[i]) <= tol - ) + coords = tmp_geometry.apply(lambda geom: np.array(geom.coords[0])).to_list() + + # Perform DBSCAN on the coordinates + db = DBSCAN(eps=tol, min_samples=1).fit(coords) - if len(close_nodes) == 1: - # if only one substation is in tolerance, then the substation is the current one iì - # Note that the node cannot be with substation_id >= 0, given the preliminary check - # at the beginning of the for loop - buses.loc[buses.index[i], "station_id"] = station_id - # update station id - station_id += 1 - else: - # several substations in tolerance - # get their ids - subset_substation_ids = buses.loc[buses.index[close_nodes], "station_id"] - # check if all substation_ids are negative (<0) - all_neg = subset_substation_ids.max() < 0 - # check if at least a substation_id is negative (<0) - some_neg = subset_substation_ids.min() < 0 - - if all_neg: - # when all substation_ids are negative, then this is a new substation id - # set the current station_id and increment the counter - buses.loc[buses.index[close_nodes], "station_id"] = station_id - station_id += 1 - elif some_neg: - # otherwise, when at least a substation_id is non-negative, then pick the first value - # and set it to all the other substations within tolerance - sub_id = -1 - for substation_id in subset_substation_ids: - if substation_id >= 0: - sub_id = substation_id - break - buses.loc[buses.index[close_nodes], "station_id"] = sub_id + # Add the cluster labels to the GeoDataFrame + buses['station_id'] = db.labels_ def set_lines_ids(lines, buses, distance_crs): From 8aeb088724eb323a6f484f088dd9e4531df65672 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Jul 2023 11:01:59 +0000 Subject: [PATCH 002/113] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_osm_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 16511cd37..0e10e4720 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -15,8 +15,8 @@ from config_osm_data import osm_clean_columns from shapely.geometry import LineString, Point from shapely.ops import linemerge, split -from tqdm import tqdm from sklearn.cluster import DBSCAN +from tqdm import tqdm logger = logging.getLogger(__name__) @@ -63,7 +63,7 @@ def set_substations_ids(buses, distance_crs, tol=5000): db = DBSCAN(eps=tol, min_samples=1).fit(coords) # Add the cluster labels to the GeoDataFrame - buses['station_id'] = db.labels_ + buses["station_id"] = db.labels_ def set_lines_ids(lines, buses, distance_crs): From 528a1868a0a6c32a1f095516de537b8f51ac98ac Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Sun, 23 Jul 2023 00:27:36 +0100 Subject: [PATCH 003/113] re-write fix_overpassing_lines (800 times faster( --- scripts/build_osm_network.py | 166 ++++++++++++++--------------------- 1 file changed, 64 insertions(+), 102 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index b89180c94..39a369aa7 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -13,8 +13,8 @@ import pandas as pd from _helpers import configure_logging, read_geojson, sets_path_to_root, to_csv_nafix from config_osm_data import osm_clean_columns -from shapely.geometry import LineString, Point -from shapely.ops import linemerge, split +from shapely.geometry import LineString, Point, MultiLineString +from shapely.ops import linemerge, nearest_points from tqdm import tqdm logger = logging.getLogger(__name__) @@ -595,131 +595,93 @@ def create_station_at_equal_bus_locations( return lines, buses -def _split_linestring_by_point(linestring, points): - """ - Function to split a linestring geometry by multiple inner points. - - Parameters - ---------- - lstring : LineString - Linestring of the line to be split - points : list - List of points to split the linestring - - Return - ------ - list_lines : list - List of linestring to split the line - """ - - list_linestrings = [linestring] - - for p in points: - # execute split to all lines and store results - temp_list = [split(l, p) for l in list_linestrings] - # nest all geometries - list_linestrings = [lstring for tval in temp_list for lstring in tval.geoms] - - return list_linestrings - - def fix_overpassing_lines(lines, buses, distance_crs, tol=1): """ - Function to avoid buses overpassing lines with no connection when the bus - is within a given tolerance from the line. + Function to snap buses to lines that are within a certain tolerance. Parameters ---------- lines : GeoDataFrame - Geodataframe of lines + GeoDataFrame containing the lines buses : GeoDataFrame - Geodataframe of substations + GeoDataFrame containing the buses + distance_crs : str + Coordinate reference system to use for distance calculations tol : float - Tolerance in meters of the distance between the substation and the line - below which the line will be split + Tolerance in meters to snap the buses to the lines + + Returns + ------- + lines : GeoDataFrame + GeoDataFrame containing the lines """ + + df_l = lines.copy() # can use lines directly without copying + # drop all columns excpet id and geometry for buses + df_p = buses[['id', 'geometry']].copy() - lines_to_add = [] # list of lines to be added - lines_to_split = [] # list of lines that have been split + # change crs to distance based + df_l = df_l.to_crs(distance_crs) + df_p = df_p.to_crs(distance_crs) - lines_epsgmod = lines.to_crs(distance_crs) - buses_epsgmod = buses.to_crs(distance_crs) - - # set tqdm options for substation ids - tqdm_kwargs_substation_ids = dict( - ascii=False, - unit=" lines", - total=lines.shape[0], - desc="Verify lines overpassing nodes ", - ) + # Buffer points to create areas for spatial join + buffer_df = gpd.GeoDataFrame(geometry=df_p.buffer(tol)) + + # Spatial join to find lines intersecting point buffers + joined = gpd.sjoin(df_l, buffer_df, how="inner", op='intersects') - for l in tqdm(lines.index, **tqdm_kwargs_substation_ids): - # bus indices being within tolerance from the line - bus_in_tol_epsg = buses_epsgmod[ - buses_epsgmod.geometry.distance(lines_epsgmod.geometry.loc[l]) <= tol - ] + # group lines by their ids + group_lines = joined.groupby('id') - # exclude endings of the lines - bus_in_tol_epsg = bus_in_tol_epsg[ - ( - ( - bus_in_tol_epsg.geometry.distance( - lines_epsgmod.geometry.loc[l].boundary.geoms[0] - ) - > tol - ) - | ( - bus_in_tol_epsg.geometry.distance( - lines_epsgmod.geometry.loc[l].boundary.geoms[1] - ) - > tol - ) - ) - ] + # iterate over the groups, TODO: change to apply + for i, group in group_lines: + line_id = group['id'].iloc[0] # pick the line id that represents the group + line_geom = df_l[df_l['id'] == line_id]['geometry'].iloc[0] - if not bus_in_tol_epsg.empty: - # add index of line to split - lines_to_split.append(l) + # number of points that intersect with the line + num_points = len(group) - buses_locs = buses.geometry.loc[bus_in_tol_epsg.index] + # get the indeces of the points that intersect with the line + points_indexes = group['index_right'].tolist() + + # get the geometries of the points that intersect with the line + multi_points = df_p.loc[points_indexes, 'geometry'].tolist() - # get new line geometries - new_geometries = _split_linestring_by_point(lines.geometry[l], buses_locs) - n_geoms = len(new_geometries) + # finda all the nearest points on the line to the points that intersect with the line + nearest_points_list = [nearest_points(line_geom, point)[0] for point in multi_points] + + # create perpendicular lines from the points that intersect with the line to the nearest points on the line + perpendicular_lines = [LineString([point, nearest_point]) for point, nearest_point in zip(multi_points, nearest_points_list)] - # create temporary copies of the line - df_append = gpd.GeoDataFrame([lines.loc[l]] * n_geoms) - # update geometries - df_append["geometry"] = new_geometries - # update name of the line - df_append["line_id"] = [ - str(df_append["line_id"].iloc[0]) + f"_{id}" for id in range(n_geoms) - ] + # split the line geom with the perpendicular lines using difference + split_line = line_geom.difference(MultiLineString(perpendicular_lines)) - lines_to_add.append(df_append) + # TODO: replace the end points of each line in the multistring with the points that intersect with the line + # unless there is a point that is intersecting the start point of the line + # in that case replace the start point with the point that is intersecting the start point of the line - if not lines_to_add: - return lines, buses + # replace the line with the split line + df_l.loc[df_l['id'] == line_id, 'geometry'] = split_line - df_to_add = gpd.GeoDataFrame(pd.concat(lines_to_add, ignore_index=True)) - df_to_add.set_crs(lines.crs, inplace=True) - df_to_add.set_index(lines.index[-1] + df_to_add.index, inplace=True) - # update length - df_to_add["length"] = df_to_add.to_crs(distance_crs).geometry.length + # explode the multilinestrings (not recommended, but inculded for completion) + # exploding the df should be done at the last step + # if an operation requires separate lines, it should be done using df.explode().apply(your_function) + # which is a lot more memory efficient + df_l = df_l.explode(index_parts=False) - # update line endings - df_to_add = line_endings_to_bus_conversion(df_to_add) + # update line endings (inculded for completion, the scope of the function should be limited to fixing overpassing lines) + # commented out due to errors in the bus conversion function + # df_l = line_endings_to_bus_conversion(df_l) - # remove original lines - lines.drop(lines_to_split, inplace=True) + # update length + df_l["length"] = df_l.to_crs(distance_crs).geometry.length - lines = gpd.GeoDataFrame( - pd.concat([lines, df_to_add], ignore_index=True).reset_index(drop=True), - crs=lines.crs, - ) + # return to original crs + df_l = df_l.to_crs(lines.crs) - return lines, buses + # buses should not be returned as they are not changed, but included for completion + return df_l, buses def force_ac_lines(df, col="tag_frequency"): From e403f6c1377f3adc402d37c636067124cf4f32a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 22 Jul 2023 23:58:35 +0000 Subject: [PATCH 004/113] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_osm_network.py | 42 ++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 39a369aa7..f17821ddc 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -13,7 +13,7 @@ import pandas as pd from _helpers import configure_logging, read_geojson, sets_path_to_root, to_csv_nafix from config_osm_data import osm_clean_columns -from shapely.geometry import LineString, Point, MultiLineString +from shapely.geometry import LineString, MultiLineString, Point from shapely.ops import linemerge, nearest_points from tqdm import tqdm @@ -609,49 +609,54 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): Coordinate reference system to use for distance calculations tol : float Tolerance in meters to snap the buses to the lines - + Returns ------- lines : GeoDataFrame GeoDataFrame containing the lines """ - - df_l = lines.copy() # can use lines directly without copying + + df_l = lines.copy() # can use lines directly without copying # drop all columns excpet id and geometry for buses - df_p = buses[['id', 'geometry']].copy() + df_p = buses[["id", "geometry"]].copy() # change crs to distance based df_l = df_l.to_crs(distance_crs) df_p = df_p.to_crs(distance_crs) - # Buffer points to create areas for spatial join + # Buffer points to create areas for spatial join buffer_df = gpd.GeoDataFrame(geometry=df_p.buffer(tol)) - + # Spatial join to find lines intersecting point buffers - joined = gpd.sjoin(df_l, buffer_df, how="inner", op='intersects') + joined = gpd.sjoin(df_l, buffer_df, how="inner", op="intersects") # group lines by their ids - group_lines = joined.groupby('id') + group_lines = joined.groupby("id") # iterate over the groups, TODO: change to apply for i, group in group_lines: - line_id = group['id'].iloc[0] # pick the line id that represents the group - line_geom = df_l[df_l['id'] == line_id]['geometry'].iloc[0] + line_id = group["id"].iloc[0] # pick the line id that represents the group + line_geom = df_l[df_l["id"] == line_id]["geometry"].iloc[0] # number of points that intersect with the line num_points = len(group) # get the indeces of the points that intersect with the line - points_indexes = group['index_right'].tolist() - + points_indexes = group["index_right"].tolist() + # get the geometries of the points that intersect with the line - multi_points = df_p.loc[points_indexes, 'geometry'].tolist() + multi_points = df_p.loc[points_indexes, "geometry"].tolist() # finda all the nearest points on the line to the points that intersect with the line - nearest_points_list = [nearest_points(line_geom, point)[0] for point in multi_points] - + nearest_points_list = [ + nearest_points(line_geom, point)[0] for point in multi_points + ] + # create perpendicular lines from the points that intersect with the line to the nearest points on the line - perpendicular_lines = [LineString([point, nearest_point]) for point, nearest_point in zip(multi_points, nearest_points_list)] + perpendicular_lines = [ + LineString([point, nearest_point]) + for point, nearest_point in zip(multi_points, nearest_points_list) + ] # split the line geom with the perpendicular lines using difference split_line = line_geom.difference(MultiLineString(perpendicular_lines)) @@ -661,8 +666,7 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # in that case replace the start point with the point that is intersecting the start point of the line # replace the line with the split line - df_l.loc[df_l['id'] == line_id, 'geometry'] = split_line - + df_l.loc[df_l["id"] == line_id, "geometry"] = split_line # explode the multilinestrings (not recommended, but inculded for completion) # exploding the df should be done at the last step From 1cb96334648a742adf5e91b95329c86cb1691338 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Sun, 23 Jul 2023 16:02:01 +0200 Subject: [PATCH 005/113] revise set_lines_ids with cKDTree --- scripts/build_osm_network.py | 97 ++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index b89180c94..63b3bf60e 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -13,6 +13,7 @@ import pandas as pd from _helpers import configure_logging, read_geojson, sets_path_to_root, to_csv_nafix from config_osm_data import osm_clean_columns +from scipy.spatial import cKDTree from shapely.geometry import LineString, Point from shapely.ops import linemerge, split from tqdm import tqdm @@ -114,12 +115,6 @@ def set_lines_ids(lines, buses, distance_crs): Function to set line buses ids to the closest bus in the list. """ # set tqdm options for set lines ids - tqdm_kwargs_line_ids = dict( - ascii=False, - unit=" lines", - total=lines.shape[0], - desc="Set line bus ids ", - ) # initialization lines["bus0"] = -1 @@ -128,55 +123,59 @@ def set_lines_ids(lines, buses, distance_crs): busesepsg = buses.to_crs(distance_crs) linesepsg = lines.to_crs(distance_crs) - for i, row in tqdm(linesepsg.iterrows(), **tqdm_kwargs_line_ids): - # select buses having the voltage level of the current line - buses_sel = busesepsg[ - (buses["voltage"] == row["voltage"]) & (buses["dc"] == row["dc"]) - ] + for key, lines_sel in linesepsg.groupby(["voltage", "dc"]): + buses_sel = busesepsg.query(f"voltage == {key[0]} and dc == {key[1]}") # find the closest node of the bus0 of the line - bus0_id = buses_sel.geometry.distance(row.geometry.boundary.geoms[0]).idxmin() - lines.loc[i, "bus0"] = buses.loc[bus0_id, "bus_id"] - - # check if the line starts exactly in the node, otherwise modify the linestring - distance_bus0 = busesepsg.geometry.loc[bus0_id].distance( - row.geometry.boundary.geoms[0] + bus0_points = np.array( + list( + lines_sel.geometry.boundary.apply( + lambda x: (x.geoms[0].x, x.geoms[0].y) + ) + ) ) - if distance_bus0 > 0.0: - # the line does not start in the node, thus modify the linestring - lines.geometry.loc[i] = linemerge( - [ - LineString( - [ - buses.geometry.loc[bus0_id], - lines.geometry.loc[i].boundary.geoms[0], - ] - ), - lines.geometry.loc[i], - ] + bus1_points = np.array( + list( + lines_sel.geometry.boundary.apply( + lambda x: (x.geoms[1].x, x.geoms[1].y) + ) ) - - # find the closest node of the bus1 of the line - bus1_id = buses_sel.geometry.distance(row.geometry.boundary.geoms[1]).idxmin() - lines.loc[i, "bus1"] = buses.loc[bus1_id, "bus_id"] - - # check if the line ends exactly in the node, otherwise modify the linestring - distance_bus1 = busesepsg.geometry.loc[bus1_id].distance( - row.geometry.boundary.geoms[1] ) - if distance_bus1 > 0.0: - # the line does not end in the node, thus modify the linestring - lines.geometry.loc[i] = linemerge( - [ - lines.geometry.loc[i], - LineString( - [ - lines.geometry.loc[i].boundary.geoms[1], - buses.geometry.loc[bus1_id], - ] - ), - ] + points_buses = np.array(list(buses_sel.geometry.apply(lambda x: (x.x, x.y)))) + + btree = cKDTree(points_buses) + dist0, idx0 = btree.query(bus0_points, k=1) # find closest points of bus0 + dist1, idx1 = btree.query(bus1_points, k=1) # find closest points of bus1 + + # set bus0 and bus1 + lines.loc[lines_sel.index, "bus0"] = buses_sel.bus_id.iloc[idx0].values + lines.loc[lines_sel.index, "bus1"] = buses_sel.bus_id.iloc[idx1].values + + # check if the line starts exactly in the bus0, otherwise modify the linestring + bus0_linestring = ( + lines.loc[lines_sel.index] + .apply( + lambda x: LineString([buses.geometry.loc[x["bus0"]], x["bus_0_coors"]]), + axis=1, ) + .set_crs(crs=lines.crs) + ) + bus1_linestring = ( + lines.loc[lines_sel.index] + .apply( + lambda x: LineString([x["bus_1_coors"], buses.geometry.loc[x["bus1"]]]), + axis=1, + ) + .set_crs(crs=lines.crs) + ) + + # update geometry with left and right linestrings to match bus0 and bus1 + lines.loc[lines_sel.index, "geometry"] = ( + lines.loc[lines_sel.index] + .union(bus0_linestring) + .union(bus1_linestring) + .apply(linemerge) + ) return lines, buses From 6454cf1f13fa29430999f281d2b372e74c79389a Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Sun, 23 Jul 2023 17:10:51 +0200 Subject: [PATCH 006/113] feature: set_lines_ids drop crs conversion --- scripts/build_osm_network.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 63b3bf60e..2fbb1a559 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -110,7 +110,7 @@ def set_substations_ids(buses, distance_crs, tol=2000): buses.loc[buses.index[close_nodes], "station_id"] = sub_id -def set_lines_ids(lines, buses, distance_crs): +def set_lines_ids(lines, buses): """ Function to set line buses ids to the closest bus in the list. """ @@ -120,11 +120,8 @@ def set_lines_ids(lines, buses, distance_crs): lines["bus0"] = -1 lines["bus1"] = -1 - busesepsg = buses.to_crs(distance_crs) - linesepsg = lines.to_crs(distance_crs) - - for key, lines_sel in linesepsg.groupby(["voltage", "dc"]): - buses_sel = busesepsg.query(f"voltage == {key[0]} and dc == {key[1]}") + for key, lines_sel in lines.groupby(["voltage", "dc"]): + buses_sel = buses.query(f"voltage == {key[0]} and dc == {key[1]}") # find the closest node of the bus0 of the line bus0_points = np.array( @@ -530,7 +527,7 @@ def merge_stations_lines_by_station_id_and_voltage( logger.info("Stage 3c/4: Specify the bus ids of the line endings") # set the bus ids to the line dataset - lines, buses = set_lines_ids(lines, buses, distance_crs) + lines, buses = set_lines_ids(lines, buses) # drop lines starting and ending in the same node lines.drop(lines[lines["bus0"] == lines["bus1"]].index, inplace=True) @@ -577,7 +574,7 @@ def create_station_at_equal_bus_locations( set_substations_ids(buses, distance_crs, tol=tol) # set the bus ids to the line dataset - lines, buses = set_lines_ids(lines, buses, distance_crs) + lines, buses = set_lines_ids(lines, buses) # update line endings lines = line_endings_to_bus_conversion(lines) From 2ecac047b09e8552d903565c2c89bf3820075f83 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Sun, 23 Jul 2023 17:10:59 +0200 Subject: [PATCH 007/113] Add release_note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 08b2d41e9..29d534e2f 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -22,6 +22,8 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t * Drop code-dependency from vresutil `PR #803 `__ +* Revise set_lines_ids with cKDTree by scipy `PR #806 `__ + PyPSA-Earth 0.2.2 ================= From ee7d02a9205d1b5dd6b3620df7ef90ea0133810f Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Mon, 24 Jul 2023 20:24:04 +0100 Subject: [PATCH 008/113] do not subset buses --- scripts/build_osm_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index f17821ddc..2e17f022c 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -618,7 +618,7 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): df_l = lines.copy() # can use lines directly without copying # drop all columns excpet id and geometry for buses - df_p = buses[["id", "geometry"]].copy() + df_p = buses.copy() # change crs to distance based df_l = df_l.to_crs(distance_crs) From 4c20bb90524ee2c48b27dca005043ee1ea0661f7 Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Mon, 24 Jul 2023 20:30:38 +0100 Subject: [PATCH 009/113] fix codespell --- scripts/build_osm_network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 2e17f022c..1e4477581 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -617,7 +617,7 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): """ df_l = lines.copy() # can use lines directly without copying - # drop all columns excpet id and geometry for buses + # drop all columns except id and geometry for buses df_p = buses.copy() # change crs to distance based @@ -641,7 +641,7 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # number of points that intersect with the line num_points = len(group) - # get the indeces of the points that intersect with the line + # get the indices of the points that intersect with the line points_indexes = group["index_right"].tolist() # get the geometries of the points that intersect with the line @@ -668,13 +668,13 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # replace the line with the split line df_l.loc[df_l["id"] == line_id, "geometry"] = split_line - # explode the multilinestrings (not recommended, but inculded for completion) + # explode the multilinestrings (not recommended, but included for completion) # exploding the df should be done at the last step # if an operation requires separate lines, it should be done using df.explode().apply(your_function) # which is a lot more memory efficient df_l = df_l.explode(index_parts=False) - # update line endings (inculded for completion, the scope of the function should be limited to fixing overpassing lines) + # update line endings (included for completion, the scope of the function should be limited to fixing overpassing lines) # commented out due to errors in the bus conversion function # df_l = line_endings_to_bus_conversion(df_l) From 349789bfa6e9ebc8c5c0f0c62bcbf68bc83e72aa Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Mon, 24 Jul 2023 21:51:08 +0100 Subject: [PATCH 010/113] id -> line_id --- scripts/build_osm_network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 1e4477581..ff67ed7d2 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -631,12 +631,12 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): joined = gpd.sjoin(df_l, buffer_df, how="inner", op="intersects") # group lines by their ids - group_lines = joined.groupby("id") + group_lines = joined.groupby("line_id") # iterate over the groups, TODO: change to apply for i, group in group_lines: - line_id = group["id"].iloc[0] # pick the line id that represents the group - line_geom = df_l[df_l["id"] == line_id]["geometry"].iloc[0] + line_id = i # pick the line id that represents the group + line_geom = df_l[df_l["line_id"] == line_id]["geometry"].iloc[0] # number of points that intersect with the line num_points = len(group) @@ -666,7 +666,7 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # in that case replace the start point with the point that is intersecting the start point of the line # replace the line with the split line - df_l.loc[df_l["id"] == line_id, "geometry"] = split_line + df_l.loc[df_l["line_id"] == line_id, "geometry"] = split_line # explode the multilinestrings (not recommended, but included for completion) # exploding the df should be done at the last step From 92c142e396a7c406697097edcfc2be15b9de8d4f Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Tue, 25 Jul 2023 16:09:27 +0100 Subject: [PATCH 011/113] crucial changes to fit workflow --- scripts/build_osm_network.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index ff67ed7d2..3f0a065fd 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -620,23 +620,29 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # drop all columns except id and geometry for buses df_p = buses.copy() + line_id_str = "line_id" + bus_id_str = "bus_id" + # change crs to distance based df_l = df_l.to_crs(distance_crs) df_p = df_p.to_crs(distance_crs) + # set index to bus_id + df_p.set_index(bus_id_str, inplace=True) + # Buffer points to create areas for spatial join - buffer_df = gpd.GeoDataFrame(geometry=df_p.buffer(tol)) + buffer_df = df_p.buffer(tol).to_frame() # Spatial join to find lines intersecting point buffers joined = gpd.sjoin(df_l, buffer_df, how="inner", op="intersects") # group lines by their ids - group_lines = joined.groupby("line_id") + group_lines = joined.groupby(line_id_str) # iterate over the groups, TODO: change to apply for i, group in group_lines: line_id = i # pick the line id that represents the group - line_geom = df_l[df_l["line_id"] == line_id]["geometry"].iloc[0] + line_geom = df_l[df_l[line_id_str] == line_id]["geometry"].iloc[0] # number of points that intersect with the line num_points = len(group) @@ -665,14 +671,15 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # unless there is a point that is intersecting the start point of the line # in that case replace the start point with the point that is intersecting the start point of the line - # replace the line with the split line - df_l.loc[df_l["line_id"] == line_id, "geometry"] = split_line + # replace the line with the split line in lines df + df_l.loc[df_l[line_id_str] == line_id, "geometry"] = split_line # explode the multilinestrings (not recommended, but included for completion) # exploding the df should be done at the last step # if an operation requires separate lines, it should be done using df.explode().apply(your_function) # which is a lot more memory efficient - df_l = df_l.explode(index_parts=False) + # df_l = df_l.explode(index_parts=False) # (recommended) + df_l = df_l.explode(ignore_index=True) # update line endings (included for completion, the scope of the function should be limited to fixing overpassing lines) # commented out due to errors in the bus conversion function @@ -684,6 +691,9 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # return to original crs df_l = df_l.to_crs(lines.crs) + # remove lines that are rings (included for completion), TODO: this should be a separate function + df_l = df_l[~df_l.geometry.is_ring].reset_index() + # buses should not be returned as they are not changed, but included for completion return df_l, buses From 6d4b6878bbf5699f47c0552e9f08d453fb97ce18 Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Thu, 27 Jul 2023 15:15:20 +0100 Subject: [PATCH 012/113] change perpendicular to bisector --- scripts/build_osm_network.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 3f0a065fd..52fd0e38f 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -653,24 +653,20 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # get the geometries of the points that intersect with the line multi_points = df_p.loc[points_indexes, "geometry"].tolist() - # finda all the nearest points on the line to the points that intersect with the line + # find all the nearest points on the line to the points that intersect with the line nearest_points_list = [ nearest_points(line_geom, point)[0] for point in multi_points ] + # reflect the points in the line to create a bisector + reflected_points_list = [Point([2 * nearest_point.x - point.x, 2 * nearest_point.y - point.y]) for point, nearest_point in zip(multi_points, nearest_points_list)] + # create perpendicular lines from the points that intersect with the line to the nearest points on the line - perpendicular_lines = [ - LineString([point, nearest_point]) - for point, nearest_point in zip(multi_points, nearest_points_list) - ] + perpendicular_lines = [LineString([point, reflected_point]) for point, reflected_point in zip(multi_points, reflected_points_list)] # split the line geom with the perpendicular lines using difference split_line = line_geom.difference(MultiLineString(perpendicular_lines)) - # TODO: replace the end points of each line in the multistring with the points that intersect with the line - # unless there is a point that is intersecting the start point of the line - # in that case replace the start point with the point that is intersecting the start point of the line - # replace the line with the split line in lines df df_l.loc[df_l[line_id_str] == line_id, "geometry"] = split_line From af79a8efc11bf1749bb88aa4a37bdd3f07b7275d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:15:40 +0000 Subject: [PATCH 013/113] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_osm_network.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 52fd0e38f..8627c80fc 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -659,10 +659,16 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): ] # reflect the points in the line to create a bisector - reflected_points_list = [Point([2 * nearest_point.x - point.x, 2 * nearest_point.y - point.y]) for point, nearest_point in zip(multi_points, nearest_points_list)] + reflected_points_list = [ + Point([2 * nearest_point.x - point.x, 2 * nearest_point.y - point.y]) + for point, nearest_point in zip(multi_points, nearest_points_list) + ] # create perpendicular lines from the points that intersect with the line to the nearest points on the line - perpendicular_lines = [LineString([point, reflected_point]) for point, reflected_point in zip(multi_points, reflected_points_list)] + perpendicular_lines = [ + LineString([point, reflected_point]) + for point, reflected_point in zip(multi_points, reflected_points_list) + ] # split the line geom with the perpendicular lines using difference split_line = line_geom.difference(MultiLineString(perpendicular_lines)) From 3941d2031a9db740e8add6c4348f83b6ea11ca67 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Wed, 16 Aug 2023 12:05:07 +0200 Subject: [PATCH 014/113] Add groupby by index and line_id by part --- scripts/build_osm_network.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 8627c80fc..5e3c592d6 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -636,13 +636,13 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): # Spatial join to find lines intersecting point buffers joined = gpd.sjoin(df_l, buffer_df, how="inner", op="intersects") - # group lines by their ids - group_lines = joined.groupby(line_id_str) + # group lines by their index + group_lines = joined.groupby(level=0) # iterate over the groups, TODO: change to apply for i, group in group_lines: - line_id = i # pick the line id that represents the group - line_geom = df_l[df_l[line_id_str] == line_id]["geometry"].iloc[0] + line_id = df_l.loc[i, line_id_str] # pick the line id that represents the group + line_geom = df_l.loc[i, "geometry"] # number of points that intersect with the line num_points = len(group) @@ -674,14 +674,17 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): split_line = line_geom.difference(MultiLineString(perpendicular_lines)) # replace the line with the split line in lines df - df_l.loc[df_l[line_id_str] == line_id, "geometry"] = split_line + df_l.loc[i, "geometry"] = split_line # explode the multilinestrings (not recommended, but included for completion) # exploding the df should be done at the last step # if an operation requires separate lines, it should be done using df.explode().apply(your_function) # which is a lot more memory efficient - # df_l = df_l.explode(index_parts=False) # (recommended) - df_l = df_l.explode(ignore_index=True) + df_l = df_l.explode(index_parts=True).reset_index() + + # revise line_id to account for part index + df_l[line_id_str] = df_l[line_id_str] + "_" + df_l["level_1"].astype(str) + df_l.drop(columns=["level_0", "level_1"], inplace=True) # update line endings (included for completion, the scope of the function should be limited to fixing overpassing lines) # commented out due to errors in the bus conversion function @@ -694,7 +697,7 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): df_l = df_l.to_crs(lines.crs) # remove lines that are rings (included for completion), TODO: this should be a separate function - df_l = df_l[~df_l.geometry.is_ring].reset_index() + df_l = df_l[~df_l.geometry.is_ring].reset_index(drop=True) # buses should not be returned as they are not changed, but included for completion return df_l, buses From 31d39558bad716e32fa24501a3e978683ecbd99b Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Wed, 16 Aug 2023 12:30:45 +0200 Subject: [PATCH 015/113] Skip buses from fix_overpassing_lines --- scripts/build_osm_network.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 5e3c592d6..6d39d7aa4 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -651,23 +651,31 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): points_indexes = group["index_right"].tolist() # get the geometries of the points that intersect with the line - multi_points = df_p.loc[points_indexes, "geometry"].tolist() + all_points = df_p.loc[points_indexes, "geometry"] + + # discard points related to the extrema points (the buses) of each line + distance_from_buses = all_points.distance(line_geom.boundary) + overpassing_points = list(all_points[distance_from_buses > tol]) + + # if no overpassing points are identified, skip iteration + if len(overpassing_points) == 0: + continue # find all the nearest points on the line to the points that intersect with the line nearest_points_list = [ - nearest_points(line_geom, point)[0] for point in multi_points + nearest_points(line_geom, point)[0] for point in overpassing_points ] # reflect the points in the line to create a bisector reflected_points_list = [ Point([2 * nearest_point.x - point.x, 2 * nearest_point.y - point.y]) - for point, nearest_point in zip(multi_points, nearest_points_list) + for point, nearest_point in zip(overpassing_points, nearest_points_list) ] # create perpendicular lines from the points that intersect with the line to the nearest points on the line perpendicular_lines = [ LineString([point, reflected_point]) - for point, reflected_point in zip(multi_points, reflected_points_list) + for point, reflected_point in zip(overpassing_points, reflected_points_list) ] # split the line geom with the perpendicular lines using difference From de9101ef6e94db2c87413183a6a653c585b08529 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Wed, 16 Aug 2023 13:25:58 +0200 Subject: [PATCH 016/113] Add escape and force str line_id --- scripts/build_osm_network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 6d39d7aa4..252921d45 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -615,6 +615,8 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): lines : GeoDataFrame GeoDataFrame containing the lines """ + if lines.empty: + return lines, buses df_l = lines.copy() # can use lines directly without copying # drop all columns except id and geometry for buses @@ -691,7 +693,9 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): df_l = df_l.explode(index_parts=True).reset_index() # revise line_id to account for part index - df_l[line_id_str] = df_l[line_id_str] + "_" + df_l["level_1"].astype(str) + df_l[line_id_str] = ( + df_l[line_id_str].astype(str) + "_" + df_l["level_1"].astype(str) + ) df_l.drop(columns=["level_0", "level_1"], inplace=True) # update line endings (included for completion, the scope of the function should be limited to fixing overpassing lines) From 975e2da9589ecb482a3326250bc55d95af1ae7c4 Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Wed, 16 Aug 2023 14:22:12 +0100 Subject: [PATCH 017/113] replace bisector with split function --- scripts/build_osm_network.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 252921d45..0cedc8478 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -14,7 +14,7 @@ from _helpers import configure_logging, read_geojson, sets_path_to_root, to_csv_nafix from config_osm_data import osm_clean_columns from shapely.geometry import LineString, MultiLineString, Point -from shapely.ops import linemerge, nearest_points +from shapely.ops import linemerge, nearest_points, split from tqdm import tqdm logger = logging.getLogger(__name__) @@ -668,20 +668,19 @@ def fix_overpassing_lines(lines, buses, distance_crs, tol=1): nearest_points(line_geom, point)[0] for point in overpassing_points ] - # reflect the points in the line to create a bisector - reflected_points_list = [ - Point([2 * nearest_point.x - point.x, 2 * nearest_point.y - point.y]) - for point, nearest_point in zip(overpassing_points, nearest_points_list) - ] + # sort the nearest points based on their distance from the start point of the line + nearest_points_list.sort(key=lambda point: line_geom.project(point)) - # create perpendicular lines from the points that intersect with the line to the nearest points on the line - perpendicular_lines = [ - LineString([point, reflected_point]) - for point, reflected_point in zip(overpassing_points, reflected_points_list) - ] + # split the line at each nearest point using the split function + split_line = [line_geom] + for point in nearest_points_list: + # Split the line at the current point + # The split function returns a GeometryCollection, so we need to convert it to a list + split_lines = split(split_line[-1], point) + split_line = split_line[:-1] + list(split_lines.geoms) - # split the line geom with the perpendicular lines using difference - split_line = line_geom.difference(MultiLineString(perpendicular_lines)) + # convert the split line to a multilinestring + split_line = MultiLineString(split_line) # replace the line with the split line in lines df df_l.loc[i, "geometry"] = split_line From bdf3e1bd27e5e12d59ebd41b8952610fbf365f68 Mon Sep 17 00:00:00 2001 From: Matin Mahmood Date: Wed, 16 Aug 2023 15:08:24 +0100 Subject: [PATCH 018/113] improve docstrings --- scripts/build_osm_network.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 0cedc8478..0620d3a94 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -597,7 +597,13 @@ def create_station_at_equal_bus_locations( def fix_overpassing_lines(lines, buses, distance_crs, tol=1): """ - Function to snap buses to lines that are within a certain tolerance. + Snap buses to lines that are within a certain tolerance. + It does this by first buffering the buses by the tolerance distance, + and then performing a spatial join to find all lines that intersect with the buffers. + For each group of lines that intersect with a buffer, the function identifies the points that overpass the line (i.e., are not snapped to the line), + and then snaps those points to the nearest point on the line. + The line is then split at each snapped point, resulting in a new set of lines that are snapped to the buses. + The function returns a GeoDataFrame containing the snapped lines, and the original GeoDataFrame containing the buses. Parameters ---------- From 4d8ff70f0df6525c276f317dc29d3f4b50bb853d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:08:47 +0000 Subject: [PATCH 019/113] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_osm_network.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 0620d3a94..42d00754b 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -597,13 +597,15 @@ def create_station_at_equal_bus_locations( def fix_overpassing_lines(lines, buses, distance_crs, tol=1): """ - Snap buses to lines that are within a certain tolerance. - It does this by first buffering the buses by the tolerance distance, - and then performing a spatial join to find all lines that intersect with the buffers. - For each group of lines that intersect with a buffer, the function identifies the points that overpass the line (i.e., are not snapped to the line), - and then snaps those points to the nearest point on the line. - The line is then split at each snapped point, resulting in a new set of lines that are snapped to the buses. - The function returns a GeoDataFrame containing the snapped lines, and the original GeoDataFrame containing the buses. + Snap buses to lines that are within a certain tolerance. It does this by + first buffering the buses by the tolerance distance, and then performing a + spatial join to find all lines that intersect with the buffers. For each + group of lines that intersect with a buffer, the function identifies the + points that overpass the line (i.e., are not snapped to the line), and then + snaps those points to the nearest point on the line. The line is then split + at each snapped point, resulting in a new set of lines that are snapped to + the buses. The function returns a GeoDataFrame containing the snapped + lines, and the original GeoDataFrame containing the buses. Parameters ---------- From 4272a22992db74b3cde6f2595f5e1a14b9b91c07 Mon Sep 17 00:00:00 2001 From: davide-f Date: Fri, 25 Aug 2023 20:31:33 +0200 Subject: [PATCH 020/113] Revise logger steps --- scripts/build_osm_network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 3d7ad3032..32bc561a8 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -469,19 +469,19 @@ def merge_stations_lines_by_station_id_and_voltage( """ logger.info( - "Stage 3a/4: Set substation ids with tolerance of %.2f km" % (tol / 1000) + "Stage 4a/5: Set substation ids with tolerance of %.2f km" % (tol / 1000) ) # set substation ids set_substations_ids(buses, distance_crs, tol=tol) - logger.info("Stage 3b/4: Merge substations with the same id") + logger.info("Stage 4b/5: Merge substations with the same id") # merge buses with same station id and voltage if not buses.empty: buses = merge_stations_same_station_id(buses) - logger.info("Stage 3c/4: Specify the bus ids of the line endings") + logger.info("Stage 4c/5: Specify the bus ids of the line endings") # set the bus ids to the line dataset lines, buses = set_lines_ids(lines, buses) @@ -494,7 +494,7 @@ def merge_stations_lines_by_station_id_and_voltage( # set substation_lv set_lv_substations(buses) - logger.info("Stage 3d/4: Add converters to lines") + logger.info("Stage 4d/5: Add converters to lines") # append fake converters # lines = pd.concat([lines, converters], ignore_index=True) From d60276cfbc318b70bcff61338c0463e30a56ba3d Mon Sep 17 00:00:00 2001 From: davide-f Date: Fri, 25 Aug 2023 20:32:27 +0200 Subject: [PATCH 021/113] Drop misimport --- scripts/build_osm_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 32bc561a8..4b510d45e 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -14,7 +14,7 @@ from _helpers import configure_logging, read_geojson, sets_path_to_root, to_csv_nafix from config_osm_data import osm_clean_columns from scipy.spatial import cKDTree -from shapely.geometry import LineString, Point, MultiLineString, Point +from shapely.geometry import LineString, MultiLineString, Point from shapely.ops import linemerge, nearest_points, split from sklearn.cluster import DBSCAN from tqdm import tqdm From 549fc48ae51382fe59e1d32a0bb9c51ab48bb1c9 Mon Sep 17 00:00:00 2001 From: davide-f Date: Sat, 26 Aug 2023 19:11:09 +0200 Subject: [PATCH 022/113] Set distance_crs for set_line_ids and drop legacy code --- scripts/build_osm_network.py | 59 ++++-------------------------------- 1 file changed, 6 insertions(+), 53 deletions(-) diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 4b510d45e..2d1a8c096 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -67,18 +67,20 @@ def set_substations_ids(buses, distance_crs, tol=5000): buses["station_id"] = db.labels_ -def set_lines_ids(lines, buses): +def set_lines_ids(lines, buses, distance_crs): """ Function to set line buses ids to the closest bus in the list. """ # set tqdm options for set lines ids + lines_d = lines.to_crs(distance_crs) + buses_d = buses.to_crs(distance_crs) # initialization lines["bus0"] = -1 lines["bus1"] = -1 - for key, lines_sel in lines.groupby(["voltage", "dc"]): - buses_sel = buses.query(f"voltage == {key[0]} and dc == {key[1]}") + for key, lines_sel in lines_d.groupby(["voltage", "dc"]): + buses_sel = buses_d.query(f"voltage == {key[0]} and dc == {key[1]}") # find the closest node of the bus0 of the line bus0_points = np.array( @@ -452,14 +454,6 @@ def set_lv_substations(buses): return buses -# Note tolerance = 0.01 means around 700m -# TODO: the current tolerance is high to avoid an issue in the Nigeria case where line 565939360-1 -# seems to be interconnected to both ends, but at the eastern one, the node is actually not connected -# another line seems to be exactly touching the node, but from the data point of view it only fly over it. -# There may be the need to split a line in several segments in the case the line is within tolerance with -# respect to a node - - def merge_stations_lines_by_station_id_and_voltage( lines, buses, geo_crs, distance_crs, tol=2000 ): @@ -484,7 +478,7 @@ def merge_stations_lines_by_station_id_and_voltage( logger.info("Stage 4c/5: Specify the bus ids of the line endings") # set the bus ids to the line dataset - lines, buses = set_lines_ids(lines, buses) + lines, buses = set_lines_ids(lines, buses, distance_crs) # drop lines starting and ending in the same node lines.drop(lines[lines["bus0"] == lines["bus1"]].index, inplace=True) @@ -507,47 +501,6 @@ def merge_stations_lines_by_station_id_and_voltage( return lines, buses -def create_station_at_equal_bus_locations( - lines, buses, geo_crs, distance_crs, tol=2000 -): - # V1. Create station_id at same bus location - # - We saw that buses are not connected exactly at one point, they are - # usually connected to a substation "area" (analysed on maps) - # - Create station_id at exactly the same location might therefore be not - # always correct - # - Though as you can see below, it might be still sometime the case. - # Examples are **station 4** (2 lines with the same voltage connect at the - # same point) and **station 23** (4 lines with two different voltages connect - # at the same point) - # TODO: Filter out the generator lines - defined as going from generator to - # the next station which is connected to a load. Excluding generator - # lines make probably sense because they are not transmission expansion - # relevant. For now we simplify and include generator lines. - - # If same location/geometry make station - bus_all = buses - - # set substation ids - set_substations_ids(buses, distance_crs, tol=tol) - - # set the bus ids to the line dataset - lines, buses = set_lines_ids(lines, buses) - - # update line endings - lines = line_endings_to_bus_conversion(lines) - - # For each station number with multiple buses make lowest voltage `substation_lv = TRUE` - set_lv_substations(bus_all) - - # TRY: Keep only buses that are not duplicated & lv_substation = True - # TODO: Check if this is necessary. What effect do duplicates have? - bus_all = bus_all[bus_all["substation_lv"] == True] - - lines = connect_stations_same_station_id(lines, buses) - - return lines, buses - - def fix_overpassing_lines(lines, buses, distance_crs, tol=1): """ Snap buses to lines that are within a certain tolerance. It does this by From c77f71638336e52ad1a66bcad232cb05cf39714c Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 10 Oct 2024 17:39:50 +0700 Subject: [PATCH 023/113] adding dev container config --- .devcontainer/devcontainer.json | 9 +++++++ .devcontainer/setup.sh | 6 +++++ .devcontainer/welcome-message.txt | 13 +++++++++ .github/.devcontainer/devcontainer.json | 33 +++++++++++++++++++++++ .github/workflows/devcontainer.yaml | 36 +++++++++++++++++++++++++ Dockerfile | 18 +++++++++++++ Dockerfile.solved | 26 ++++++++++++++++++ config.tutorial.yaml | 1 + 8 files changed, 142 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/setup.sh create mode 100644 .devcontainer/welcome-message.txt create mode 100644 .github/.devcontainer/devcontainer.json create mode 100644 .github/workflows/devcontainer.yaml create mode 100644 Dockerfile create mode 100644 Dockerfile.solved diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..81e1605d9 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,9 @@ +// For format details, see https://aka.ms/devcontainer.json. +{ + "name": "pypsa earth dev", + "image": "ghcr.io/drifter089/pypsa-earth:latest", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces,type=bind,consistency=cached", + "initializeCommand": "docker pull ghcr.io/drifter089/pypsa-earth:latest", + "workspaceFolder": "/workspaces", + "postAttachCommand": "bash .devcontainer/setup.sh" +} diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100644 index 000000000..e2fc1a3e8 --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# mv ../pypsa-eur/* ./ +# echo 'solved tutorial files are copied to workspace' + +cat .devcontainer/welcome-message.txt \ No newline at end of file diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt new file mode 100644 index 000000000..a0c311068 --- /dev/null +++ b/.devcontainer/welcome-message.txt @@ -0,0 +1,13 @@ + + +👋 Welcome to pypsa-eur contribution in a dev container! + Works in VS Code, or in docker using the devcontainer cli + +See https://github.com/drifter089/pypsa-eur/blob/master/README.md for guidance on debugging and contributing to pypsa-eur + + +For details about dev containers see https://containers.dev + +The configuration for the dev container is in the .github/.devcontainer folder. + +🔍 To explore VS Code to its fullest, search using the Command Palette (Cmd/Ctrl + Shift + P or F1).+ \ No newline at end of file diff --git a/.github/.devcontainer/devcontainer.json b/.github/.devcontainer/devcontainer.json new file mode 100644 index 000000000..fda35213c --- /dev/null +++ b/.github/.devcontainer/devcontainer.json @@ -0,0 +1,33 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu +{ + "name": "pypsa earth dev", + "build": { + "dockerfile": "../../Dockerfile" + }, + + "features": { + "ghcr.io/devcontainers-contrib/features/bash-command:1": {}, + "ghcr.io/eliises/devcontainer-features/bash-profile:1": {} + }, + "customizations": { + "vscode": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/bash" + } + }, + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-azuretools.vscode-docker", + "ms-toolsai.jupyter", + "zainchen.json", + "tomoki1207.pdf", + "grapecity.gc-excelviewer" + ] + } + }, + + "postCreateCommand": "python -m pip install --upgrade debugpy" +} diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml new file mode 100644 index 000000000..ec48a223e --- /dev/null +++ b/.github/workflows/devcontainer.yaml @@ -0,0 +1,36 @@ +name: Dev Container Build and Push Image + +on: + workflow_dispatch: + push: + branches: + - "main" + tags: + - "v*.*.*" + pull_request: + branches: [main] + + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - + name: Checkout + id: checkout + uses: actions/checkout@v1 + - + name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.REGISTRY_TOKEN }} + - + name: Pre-build dev container image + uses: devcontainers/ci@v0.2 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }} + push: always \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..6c8c64296 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM condaforge/mambaforge + +RUN conda update -n base conda +RUN conda install -n base conda-libmamba-solver +RUN conda config --set solver libmamba + +WORKDIR /pypsa-earth + +COPY ./envs ./envs + +RUN conda env create --file envs/environment.yaml + +RUN rm -r envs + +RUN echo "source activate pypsa-earth" > ~/.bashrc +ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH + +ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/Dockerfile.solved b/Dockerfile.solved new file mode 100644 index 000000000..f832c7a2b --- /dev/null +++ b/Dockerfile.solved @@ -0,0 +1,26 @@ +FROM condaforge/mambaforge + +RUN conda update -n base conda +RUN conda install -n base conda-libmamba-solver +RUN conda config --set solver libmamba + +RUN apt-get update && apt-get install -y git + +WORKDIR /pypsa-earth + +COPY . . + +RUN conda env create --file envs/environment.yaml + +RUN echo "source activate pypsa-earth" > ~/.bashrc +ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH + +RUN /bin/bash -c "source activate pypsa-earth && snakemake -j 1 solve_all_networks" + +RUN git ls-files -z | xargs -0 rm + +RUN find . -type d -empty -delete + +RUN rm -rf scripts config data .snakemake .git + +ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 0c98bb152..02ef557aa 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,6 +9,7 @@ tutorial: true countries: ["NG", "BJ"] enable: + retrieve_databundle: true build_natura_raster: true progress_bar: false From b076ea3620b3970b204d6e910de893b9b9cc3639 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 10 Oct 2024 11:13:36 +0000 Subject: [PATCH 024/113] updated docekr file --- Dockerfile | 4 ++-- Dockerfile.solved | 26 -------------------------- config.tutorial.yaml | 1 - 3 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 Dockerfile.solved diff --git a/Dockerfile b/Dockerfile index 6c8c64296..200dfd4ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,11 +8,11 @@ WORKDIR /pypsa-earth COPY ./envs ./envs -RUN conda env create --file envs/environment.yaml +RUN conda env create -n pypsa-earth --file envs/environment.yaml RUN rm -r envs -RUN echo "source activate pypsa-earth" > ~/.bashrc +RUN echo "conda activate pypsa-earth" > ~/.bashrc ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/Dockerfile.solved b/Dockerfile.solved deleted file mode 100644 index f832c7a2b..000000000 --- a/Dockerfile.solved +++ /dev/null @@ -1,26 +0,0 @@ -FROM condaforge/mambaforge - -RUN conda update -n base conda -RUN conda install -n base conda-libmamba-solver -RUN conda config --set solver libmamba - -RUN apt-get update && apt-get install -y git - -WORKDIR /pypsa-earth - -COPY . . - -RUN conda env create --file envs/environment.yaml - -RUN echo "source activate pypsa-earth" > ~/.bashrc -ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH - -RUN /bin/bash -c "source activate pypsa-earth && snakemake -j 1 solve_all_networks" - -RUN git ls-files -z | xargs -0 rm - -RUN find . -type d -empty -delete - -RUN rm -rf scripts config data .snakemake .git - -ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 02ef557aa..0c98bb152 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,7 +9,6 @@ tutorial: true countries: ["NG", "BJ"] enable: - retrieve_databundle: true build_natura_raster: true progress_bar: false From 96d333b3714cd28ceca6a2c0b046fd3cedd47bf8 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Thu, 10 Oct 2024 19:58:13 +0700 Subject: [PATCH 025/113] updated docker file --- Dockerfile | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 200dfd4ed..ba8bd26e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,15 +4,25 @@ RUN conda update -n base conda RUN conda install -n base conda-libmamba-solver RUN conda config --set solver libmamba +RUN apt-get update && apt-get install -y bash + WORKDIR /pypsa-earth -COPY ./envs ./envs +COPY ./envs ./temp + +RUN conda env create -n pypsa-earth -f temp/environment.yaml + +RUN conda init bash -RUN conda env create -n pypsa-earth --file envs/environment.yaml +RUN touch ~/.bashrc && echo "conda activate pypsa-earth" >> ~/.bashrc -RUN rm -r envs +SHELL ["/bin/bash", "--login", "-c"] -RUN echo "conda activate pypsa-earth" > ~/.bashrc ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH -ENTRYPOINT ["tail", "-f", "/dev/null"] +RUN rm -r temp + +RUN conda clean -afy && \ + rm -rf /tmp/* + +CMD ["bash"] From 151bc2d11bcd13a64a4d4c80a1c5018013dc525a Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 14 Oct 2024 11:52:18 +0100 Subject: [PATCH 026/113] update: include java --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index ba8bd26e8..6f5b099d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,8 @@ SHELL ["/bin/bash", "--login", "-c"] ENV PATH /opt/conda/envs/pypsa-earth/bin:$PATH +RUN conda install conda-forge::openjdk -y + RUN rm -r temp RUN conda clean -afy && \ From 05cf1c49c93fec0529165da105746576525022c1 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 14 Oct 2024 21:12:49 +0100 Subject: [PATCH 027/113] docker docs init --- doc/docker_containers.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/docker_containers.rst diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst new file mode 100644 index 000000000..556ef61ba --- /dev/null +++ b/doc/docker_containers.rst @@ -0,0 +1,7 @@ +# Running PyPSA-Earth in Docker containers + +## How to use Docker containers to run PyPSA-Earth. +- install Docker +- git clone the pypsa-earth repository +- open the pypsa-earth with VSCode +- rebuild and open the repo in a container \ No newline at end of file From 0d65aac21c8e863cc72e33b8d8bed23e27240d39 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 14 Oct 2024 21:17:14 +0100 Subject: [PATCH 028/113] review welcome message for new users --- .devcontainer/welcome-message.txt | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt index a0c311068..9fe33b7d6 100644 --- a/.devcontainer/welcome-message.txt +++ b/.devcontainer/welcome-message.txt @@ -1,13 +1,21 @@ -👋 Welcome to pypsa-eur contribution in a dev container! - Works in VS Code, or in docker using the devcontainer cli -See https://github.com/drifter089/pypsa-eur/blob/master/README.md for guidance on debugging and contributing to pypsa-eur +👋 Welcome to the PyPSA-Earth Development Environment! -For details about dev containers see https://containers.dev +We’re excited to have you here! This setup allows you to contribute to PyPSA-Earth effortlessly using a development container in VS Code. -The configuration for the dev container is in the .github/.devcontainer folder. +📖 Getting Started for New Users -🔍 To explore VS Code to its fullest, search using the Command Palette (Cmd/Ctrl + Shift + P or F1).+ \ No newline at end of file + • For a step-by-step guide on setting up your environment, debugging, and making your first contribution, refer to the PyPSA-Earth README. It covers everything you need to know as a newcomer. + • The configuration files for the development container are located in the .github/.devcontainer folder. + +💡 Tips for New Users + + • Make the most of VS Code by using the Command Palette (Cmd/Ctrl + Shift + P or F1) for quick access to features and commands. + • If you’re new to development containers, learn the basics at containers.dev. + +🚀 Start Exploring and Happy Coding! + +Don’t hesitate to reach out if you need help—our community is here to support you. \ No newline at end of file From a3d22ca92c7bfdb4c7af678f5228426984a4d8ee Mon Sep 17 00:00:00 2001 From: drifter089 Date: Mon, 21 Oct 2024 16:12:35 +0700 Subject: [PATCH 029/113] running tutorial before push --- .github/.devcontainer/test.sh | 5 +++++ .github/workflows/devcontainer.yaml | 23 ++++++++++++++++------- Dockerfile | 2 +- config.tutorial.yaml | 3 ++- 4 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 .github/.devcontainer/test.sh diff --git a/.github/.devcontainer/test.sh b/.github/.devcontainer/test.sh new file mode 100644 index 000000000..c4ac3f290 --- /dev/null +++ b/.github/.devcontainer/test.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cp config.tutorial.yaml config.yaml + +snakemake -j 1 solve_all_networks \ No newline at end of file diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml index ec48a223e..1d689e233 100644 --- a/.github/workflows/devcontainer.yaml +++ b/.github/workflows/devcontainer.yaml @@ -27,10 +27,19 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.REGISTRY_TOKEN }} - - name: Pre-build dev container image - uses: devcontainers/ci@v0.2 - with: - subFolder: .github - imageName: ghcr.io/${{ github.repository }} - cacheFrom: ghcr.io/${{ github.repository }} - push: always \ No newline at end of file + name: Build Dev Container Image + uses: devcontainers/ci@v0.2 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }} + push: false + + - name: Test Container Image + run: | + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone https://github.com/drifter089/pypsa-earth/tree/devContainers && cd pypsa-earth && bash .github/.devcontainer/test.sh" + + - name: Push Container Image + if: ${{ success() }} + run: | + docker push ghcr.io/${{ github.repository }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6f5b099d6..343661abd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ RUN conda update -n base conda RUN conda install -n base conda-libmamba-solver RUN conda config --set solver libmamba -RUN apt-get update && apt-get install -y bash +RUN apt-get update && apt-get install -y bash git WORKDIR /pypsa-earth diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 0c98bb152..fb48057c9 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -6,9 +6,10 @@ version: 0.4.1 tutorial: true -countries: ["NG", "BJ"] +countries: ["NG"] enable: + retrieve_databundle: true build_natura_raster: true progress_bar: false From 952583a5cd53d1520c1066774e7194bb1a1b440f Mon Sep 17 00:00:00 2001 From: drifter089 Date: Mon, 21 Oct 2024 16:25:09 +0700 Subject: [PATCH 030/113] fixing clone command in CI --- .github/workflows/devcontainer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml index 1d689e233..67c172da8 100644 --- a/.github/workflows/devcontainer.yaml +++ b/.github/workflows/devcontainer.yaml @@ -37,7 +37,7 @@ jobs: - name: Test Container Image run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone https://github.com/drifter089/pypsa-earth/tree/devContainers && cd pypsa-earth && bash .github/.devcontainer/test.sh" + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - name: Push Container Image if: ${{ success() }} From 9a1fb06323a9cd43dfe379898ebb75f9df28af8e Mon Sep 17 00:00:00 2001 From: drifter089 Date: Mon, 21 Oct 2024 17:04:58 +0700 Subject: [PATCH 031/113] fixing variable value --- .github/workflows/devcontainer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml index 67c172da8..173b2b252 100644 --- a/.github/workflows/devcontainer.yaml +++ b/.github/workflows/devcontainer.yaml @@ -33,7 +33,7 @@ jobs: subFolder: .github imageName: ghcr.io/${{ github.repository }} cacheFrom: ghcr.io/${{ github.repository }} - push: false + push: never - name: Test Container Image run: | From ad189c5391cc87e3cbb19483509912ad1d82354f Mon Sep 17 00:00:00 2001 From: Yerbol Akhmetov <113768325+yerbol-akhmetov@users.noreply.github.com> Date: Wed, 16 Oct 2024 01:22:50 +0500 Subject: [PATCH 032/113] Fix issue of adding H2 store in `add_extra_components` rule (#1134) * fix issue of adding H2 store and make H2 Store in AC buses * add release notes * Add H2 Stores to both AC and DC buses --- doc/release_notes.rst | 2 ++ scripts/add_extra_components.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 1ea2f5393..470e9743d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -26,6 +26,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `_ +* Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `_ + PyPSA-Earth 0.4.1 ================= diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index cab8195df..2c317c305 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -104,7 +104,7 @@ def attach_stores(n, costs, config): _add_missing_carriers_from_costs(n, costs, carriers) - buses_i = n.buses.query("carrier == 'AC'").index + buses_i = n.buses.index bus_sub_dict = {k: n.buses[k].values for k in ["x", "y", "country"]} if "H2" in carriers: From fffe9c796a433b68200964972e1ffddd252fa13f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:24:48 +0200 Subject: [PATCH 033/113] [pre-commit.ci] pre-commit autoupdate (#1131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/psf/black: 24.8.0 → 24.10.0](https://github.com/psf/black/compare/24.8.0...24.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9fd50966b..54c3bcd16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ exclude: ^(LICENSES) repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer @@ -49,7 +49,7 @@ repos: # Formatting with "black" coding style - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 24.10.0 hooks: # Format Python files - id: black From ed8e300dcc1a7217edddbf7259c3b5f2a0267aa5 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Wed, 16 Oct 2024 12:56:09 +0200 Subject: [PATCH 034/113] Add a restriction to rasterio version (#1146) --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index d98f772c3..7da885a73 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -56,7 +56,7 @@ dependencies: # GIS dependencies: - cartopy - descartes -- rasterio!=1.2.10 +- rasterio!=1.2.10, <=1.3.11 - rioxarray # Plotting From 95e0aab64bdfa906663a937af8ad18828ba2418a Mon Sep 17 00:00:00 2001 From: Yerbol Akhmetov <113768325+yerbol-akhmetov@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:39:47 +0500 Subject: [PATCH 035/113] Enable configfile specification for mock_snakemake (#1135) * enable configfile for mock_snakemake * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add release notes * drop else condition --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- doc/release_notes.rst | 2 ++ scripts/_helpers.py | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 470e9743d..a900e5860 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -28,6 +28,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Fix the mismatch between buses and x, y locations while creating H2 Stores `PR #1134 `_ +* Enable configfile specification for mock_snakemake `PR #1135 `_ + PyPSA-Earth 0.4.1 ================= diff --git a/scripts/_helpers.py b/scripts/_helpers.py index e3ceb2adf..76bf0268d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -522,7 +522,9 @@ def get_aggregation_strategies(aggregation_strategies): return bus_strategies, generator_strategies -def mock_snakemake(rulename, root_dir=None, submodule_dir=None, **wildcards): +def mock_snakemake( + rulename, root_dir=None, submodule_dir=None, configfile=None, **wildcards +): """ This function is expected to be executed from the "scripts"-directory of " the snakemake project. It returns a snakemake.script.Snakemake object, @@ -534,6 +536,8 @@ def mock_snakemake(rulename, root_dir=None, submodule_dir=None, **wildcards): ---------- rulename: str name of the rule for which the snakemake object should be generated + configfile: str + path to config file to be used in mock_snakemake wildcards: keyword arguments fixing the wildcards. Only necessary if wildcards are needed. @@ -566,9 +570,17 @@ def mock_snakemake(rulename, root_dir=None, submodule_dir=None, **wildcards): if os.path.exists(p): snakefile = p break + + if isinstance(configfile, str): + with open(configfile, "r") as file: + configfile = yaml.safe_load(file) + workflow = sm.Workflow( - snakefile, overwrite_configfiles=[], rerun_triggers=[] - ) # overwrite_config=config + snakefile, + overwrite_configfiles=[], + rerun_triggers=[], + overwrite_config=configfile, + ) workflow.include(snakefile) workflow.global_resources = {} try: From fb5c61ba73307a86365c670694c07fb47e8f6375 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Tue, 22 Oct 2024 12:06:23 +0100 Subject: [PATCH 036/113] update: set env file required for cluster network --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 343661abd..b2b171658 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,4 +27,7 @@ RUN rm -r temp RUN conda clean -afy && \ rm -rf /tmp/* +ENV LD_PRELOAD=/opt/conda/envs/pypsa-earth/lib/python3.10/site-packages/sklearn/utils/../../../../libgomp.so.1 + CMD ["bash"] + From 4fc5fc362a8240ff38c5345c846d1156ff46335f Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 28 Oct 2024 15:21:59 +0100 Subject: [PATCH 037/113] update: include mac into the CI --- .../{devcontainer.yaml => devcontainer.yml} | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) rename .github/workflows/{devcontainer.yaml => devcontainer.yml} (88%) diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yml similarity index 88% rename from .github/workflows/devcontainer.yaml rename to .github/workflows/devcontainer.yml index 173b2b252..a41bf6f4c 100644 --- a/.github/workflows/devcontainer.yaml +++ b/.github/workflows/devcontainer.yml @@ -13,9 +13,17 @@ on: jobs: build-and-push: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: + - [ubuntu-latest] + - [macos-latest] + + runs-on: ${{ matrix.os }} + steps: - - + - name: Checkout id: checkout uses: actions/checkout@v1 From 15a36427652a9142c5d828f8322c955ee7814b59 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 29 Oct 2024 13:03:45 +0100 Subject: [PATCH 038/113] Embed users list to the read-the-docs structure --- doc/index.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index 0f7e5c20c..ae8259b66 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -186,6 +186,17 @@ Documentation monte_carlo +**Applications** + +* :doc:`users_list` + +.. toctree:: + :hidden: + :maxdepth: 2 + :caption: Applications + + users_list + **Help and References** * :doc:`release_notes` From 5e4c0c66b59c0dc48fa1818fd0873bc63a8c25a0 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 29 Oct 2024 13:04:05 +0100 Subject: [PATCH 039/113] Draft users list --- doc/users_list.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/users_list.rst diff --git a/doc/users_list.rst b/doc/users_list.rst new file mode 100644 index 000000000..ca4cefdc9 --- /dev/null +++ b/doc/users_list.rst @@ -0,0 +1,20 @@ +.. SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +.. +.. SPDX-License-Identifier: CC-BY-4.0 + +.. PyPSA meets Earth documentation master file, created by + sphinx-quickstart on Sat May 15 22:52:54 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Energy transition pathways for Nigeria (Research paper) + +EDF project + +Implications of the national decarbonisaiton goals for power system of Kazakhstan (project of Open Energy Transition + Agora Energiewende) + +Possible implementation of energy transition in Saudi Arabia (MSc Thesis in Edinburgh University) + +Power system modeling for Korea considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario ("Energy Innovation Lab", Korea University https://energyinnovation.korea.ac.kr/) + +Pakistan From 80672c4657cb2f48b9070aa490f7196057209e07 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 29 Oct 2024 13:29:52 +0100 Subject: [PATCH 040/113] Add details for Korea university --- doc/users_list.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index ca4cefdc9..1d0a4cbda 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -15,6 +15,6 @@ Implications of the national decarbonisaiton goals for power system of Kazakhsta Possible implementation of energy transition in Saudi Arabia (MSc Thesis in Edinburgh University) -Power system modeling for Korea considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario ("Energy Innovation Lab", Korea University https://energyinnovation.korea.ac.kr/) +Power system modeling for Korea to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) Pakistan From b54eacd5a86848c0d5df4a6602874db3374cc827 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:00:08 +0700 Subject: [PATCH 041/113] adding user as secret --- .github/workflows/devcontainer.yaml | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/devcontainer.yaml diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml new file mode 100644 index 000000000..d38dd1fe0 --- /dev/null +++ b/.github/workflows/devcontainer.yaml @@ -0,0 +1,45 @@ +name: Dev Container Build and Push Image + +on: + workflow_dispatch: + push: + branches: + - "main" + tags: + - "v*.*.*" + pull_request: + branches: [main] + + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - + name: Checkout + id: checkout + uses: actions/checkout@v1 + - + name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + - + name: Build Dev Container Image + uses: devcontainers/ci@v0.2 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }} + push: never + + - name: Test Container Image + run: | + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" + + - name: Push Container Image + if: ${{ success() }} + run: | + docker push ghcr.io/${{ github.repository }} \ No newline at end of file From 5a6e21008917fd040752fe01fd4a294e527c7581 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:11:33 +0700 Subject: [PATCH 042/113] delete duplicate file and add user as secret --- .github/workflows/devcontainer.yaml | 45 ----------------------------- .github/workflows/devcontainer.yml | 2 +- 2 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 .github/workflows/devcontainer.yaml diff --git a/.github/workflows/devcontainer.yaml b/.github/workflows/devcontainer.yaml deleted file mode 100644 index d38dd1fe0..000000000 --- a/.github/workflows/devcontainer.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Dev Container Build and Push Image - -on: - workflow_dispatch: - push: - branches: - - "main" - tags: - - "v*.*.*" - pull_request: - branches: [main] - - -jobs: - build-and-push: - runs-on: ubuntu-latest - steps: - - - name: Checkout - id: checkout - uses: actions/checkout@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Build Dev Container Image - uses: devcontainers/ci@v0.2 - with: - subFolder: .github - imageName: ghcr.io/${{ github.repository }} - cacheFrom: ghcr.io/${{ github.repository }} - push: never - - - name: Test Container Image - run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - - - name: Push Container Image - if: ${{ success() }} - run: | - docker push ghcr.io/${{ github.repository }} \ No newline at end of file diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index a41bf6f4c..16ab1c038 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -32,7 +32,7 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image From 8abaa7b2560b055fd4d7d5dc90cb7a6296ebbaca Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:21:57 +0700 Subject: [PATCH 043/113] setup docekr on macos --- .github/workflows/devcontainer.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 16ab1c038..71db12c9b 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -27,13 +27,17 @@ jobs: name: Checkout id: checkout uses: actions/checkout@v1 + + - name: Setup Docker on macOS + if: runner.os == 'macOS' + uses: douglascamata/setup-docker-macos-action@v1-alpha - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_TOKEN }} + name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image uses: devcontainers/ci@v0.2 From a2f570e9bc975c9e59781f6086d59191601bcd20 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:24:57 +0700 Subject: [PATCH 044/113] trying colima for docekr setup on mac --- .github/workflows/devcontainer.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 71db12c9b..b5b1d4f7c 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -28,9 +28,13 @@ jobs: id: checkout uses: actions/checkout@v1 - - name: Setup Docker on macOS + - name: Setup Colima (Docker alternative for macOS) if: runner.os == 'macOS' - uses: douglascamata/setup-docker-macos-action@v1-alpha + run: | + brew install colima docker + colima start --cpu 2 --memory 4 + # Wait for Docker to be ready + while ! docker info >/dev/null 2>&1; do sleep 1; done - name: Login to GitHub Container Registry uses: docker/login-action@v2 From 9f4a0bfe095bf6ee2ca184fe9cb0b81c9a5121c1 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:33:37 +0700 Subject: [PATCH 045/113] adding mac docker setup --- .github/workflows/devcontainer.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index b5b1d4f7c..dd2e49843 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -18,7 +18,7 @@ jobs: matrix: os: - [ubuntu-latest] - - [macos-latest] + - [ macos-13 ] runs-on: ${{ matrix.os }} @@ -28,15 +28,13 @@ jobs: id: checkout uses: actions/checkout@v1 - - name: Setup Colima (Docker alternative for macOS) - if: runner.os == 'macOS' + - name: Setup docker (missing on MacOS) + if: runner.os == 'macos' run: | - brew install colima docker - colima start --cpu 2 --memory 4 - # Wait for Docker to be ready - while ! docker info >/dev/null 2>&1; do sleep 1; done - - - name: Login to GitHub Container Registry + brew install docker + colima start + + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io From 928195e520890b0f9a0deaf0293c7e25eb6bb4de Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:38:06 +0700 Subject: [PATCH 046/113] installing colima on mac --- .github/workflows/devcontainer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index dd2e49843..8de44c3a8 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -32,6 +32,7 @@ jobs: if: runner.os == 'macos' run: | brew install docker + brew install colima colima start - name: Login to GitHub Container Registry From d50db33d9eeb20d0e5e3657c1198ec420367d787 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 01:54:32 +0700 Subject: [PATCH 047/113] disable buildkit for docker --- .github/workflows/devcontainer.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 8de44c3a8..d1ae2402a 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -43,6 +43,8 @@ jobs: password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image + env: + DOCKER_BUILDKIT: ${{ runner.os == 'macOS' && '0' || '' }} uses: devcontainers/ci@v0.2 with: subFolder: .github @@ -51,6 +53,8 @@ jobs: push: never - name: Test Container Image + env: + DOCKER_BUILDKIT: ${{ runner.os == 'macOS' && '0' || '' }} run: | docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" From 781f8ee62cbfb67ffc6130aeab41f650c132ea7a Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 30 Oct 2024 02:13:18 +0700 Subject: [PATCH 048/113] add buildx plugin --- .github/workflows/devcontainer.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index d1ae2402a..ca1304bf8 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -31,9 +31,12 @@ jobs: - name: Setup docker (missing on MacOS) if: runner.os == 'macos' run: | - brew install docker + brew install docker docker-buildx brew install colima colima start + while ! docker info >/dev/null 2>&1; do sleep 1; done + mkdir -p ~/.docker/cli-plugins + ln -sfn $(which docker-buildx) ~/.docker/cli-plugins/docker-buildx - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -43,8 +46,6 @@ jobs: password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image - env: - DOCKER_BUILDKIT: ${{ runner.os == 'macOS' && '0' || '' }} uses: devcontainers/ci@v0.2 with: subFolder: .github @@ -53,8 +54,6 @@ jobs: push: never - name: Test Container Image - env: - DOCKER_BUILDKIT: ${{ runner.os == 'macOS' && '0' || '' }} run: | docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" From d6188d290f961f52efda6e4a699552240f66dde5 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Wed, 6 Nov 2024 06:29:17 +0100 Subject: [PATCH 049/113] include documentation for using docker container --- doc/docker_containers.rst | 52 ++++++++++++++++++++++++++++++++++----- doc/index.rst | 2 ++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index 556ef61ba..65436a8ff 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -1,7 +1,47 @@ -# Running PyPSA-Earth in Docker containers -## How to use Docker containers to run PyPSA-Earth. -- install Docker -- git clone the pypsa-earth repository -- open the pypsa-earth with VSCode -- rebuild and open the repo in a container \ No newline at end of file +.. _docker_containers: + +Alternate Installation with Docker +=============================================== + +This is an alternative way to create a development environment for PyPSA-Earth. This method is useful for users who are not familiar with programming or Python, or who do not want to install Python on their local machine. It uses Docker containers to create a development environment for PyPSA-Earth. + +This section provides a step-by-step guide on how to set up and use Docker containers to run PyPSA-Earth. + +Steps: + +1. Install Docker: Follow the instructions for your operating system: + - `Windows `_ + - `Linux `_ + - `MacOS `_ + + Ensure Docker is installed on your system. + +2. Install GitHub Desktop for your OS `here `_. + + + - Open GitHub Desktop. + - Click on "File" in the top left corner. + - Click on "Clone Repository". + - Paste the following URL in the URL field: + + .. code:: bash + + https://github.com/drifter089/pypsa-earth.git + + - Click on "Clone". + - Choose the location where you want to save the repository. + - Click on "Current Branch: main" and select `devContainers`. + - Click on "Open in Visual Studio Code". + + The repository will be cloned to your local machine. + +4. Rebuild and open in a container: + - Open the repository in VSCode. + - Click on the green icon in the bottom left corner of the VSCode window. + - Click on "Reopen in Container". + - Wait for the container to build and open the repository in the container. + + The environment will be ready for use. + + \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst index 0f7e5c20c..ca34d3271 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -128,6 +128,7 @@ Documentation * :doc:`introduction` * :doc:`installation` +* :doc:`docker_containers` * :doc:`short_tutorial` * :doc:`tutorial` * :doc:`data_workflow` @@ -140,6 +141,7 @@ Documentation introduction installation + docker_containers short_tutorial tutorial data_workflow From 49e5c5f3ec005459b4c0919688fa2a2e9d9dcec1 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Wed, 6 Nov 2024 06:30:31 +0100 Subject: [PATCH 050/113] update docker docs --- doc/docker_containers.rst | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index 65436a8ff..bc0ea71af 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -11,37 +11,38 @@ This section provides a step-by-step guide on how to set up and use Docker conta Steps: 1. Install Docker: Follow the instructions for your operating system: - - `Windows `_ - - `Linux `_ - - `MacOS `_ + +* `Windows `_ +* `Linux `_ +* `MacOS `_ Ensure Docker is installed on your system. 2. Install GitHub Desktop for your OS `here `_. +3. Clone the repository: + * Open GitHub Desktop. + * Click on "File" in the top left corner. + * Click on "Clone Repository". + * Paste the following URL in the URL field: - - Open GitHub Desktop. - - Click on "File" in the top left corner. - - Click on "Clone Repository". - - Paste the following URL in the URL field: - - .. code:: bash + .. code:: bash - https://github.com/drifter089/pypsa-earth.git + https://github.com/drifter089/pypsa-earth.git - - Click on "Clone". - - Choose the location where you want to save the repository. - - Click on "Current Branch: main" and select `devContainers`. - - Click on "Open in Visual Studio Code". + * Click on "Clone". + * Choose the location where you want to save the repository. + * Click on "Current Branch: main" and select `devContainers`. + * Click on "Open in Visual Studio Code". The repository will be cloned to your local machine. 4. Rebuild and open in a container: - - Open the repository in VSCode. - - Click on the green icon in the bottom left corner of the VSCode window. - - Click on "Reopen in Container". - - Wait for the container to build and open the repository in the container. + * Open the repository in VSCode. + * Click on the green icon in the bottom left corner of the VSCode window. + * Click on "Reopen in Container". + * Wait for the container to build and open the repository in the container. - The environment will be ready for use. + The environment will be ready for use. You can now run PyPSA-Earth in the container. \ No newline at end of file From 4117756d8e68174cfbe7b959e62dd6daeadc46da Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Thu, 7 Nov 2024 17:58:55 +0100 Subject: [PATCH 051/113] include docker readme --- Docker.MD | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 Docker.MD diff --git a/Docker.MD b/Docker.MD new file mode 100644 index 000000000..45e783e77 --- /dev/null +++ b/Docker.MD @@ -0,0 +1,49 @@ +# PyPSA-Earth Development Environment Setup with Docker + +This guide provides an alternative way to set up a development environment for **PyPSA-Earth** using Docker containers. This method is particularly useful for users who are not familiar with programming or Python, or who prefer not to install Python directly on their local machine. Using Docker simplifies setup by creating a self-contained environment for PyPSA-Earth. + +## Prerequisites + +### 1. Install Docker + +Ensure Docker is installed on your system. Follow the instructions for your operating system: + +- **Windows**: [Docker for Windows](https://docs.docker.com/desktop/install/windows-install/) +- **Linux**: [Docker for Linux](https://docs.docker.com/desktop/install/linux/) +- **MacOS**: [Docker for MacOS](https://docs.docker.com/desktop/install/mac-install/) + +### 2. Install GitHub Desktop + +You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install GitHub Desktop for your operating system from [here](https://desktop.github.com/download/). + +## Steps to Set Up PyPSA-Earth with Docker + +### Step 1: Clone the Repository + +1. Open **GitHub Desktop**. +2. Go to **File** > **Clone Repository**. +3. Paste the following URL in the URL field: + + ```bash + https://github.com/drifter089/pypsa-earth.git + ``` + +4. Click on **Clone**. +5. Choose the location where you want to save the repository on your local machine. +6. After cloning, click on **Current Branch: main** and select `devContainers`. +7. Click on **Open in Visual Studio Code**. + +### Step 2: Rebuild and Open in Container + +1. Open the cloned repository in **VSCode**. +2. Click on the green icon in the bottom left corner of the VSCode window. +3. Select **Reopen in Container**. +4. Wait for the container to build and for the repository to open in the container. + +Once these steps are completed, your development environment will be ready, and you can start using **PyPSA-Earth** in the Docker container. + +--- + +You are now all set up! You can use the development environment to explore PyPSA-Earth and make modifications as needed within the Docker container. + +You can start running the tutorial [here](https://pypsa-earth.readthedocs.io/en/latest/short_tutorial.html) \ No newline at end of file From 9a5dc0fcc4ba83eff8fd54e837dbd29abf8f0b6a Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 16:44:37 +0700 Subject: [PATCH 052/113] updated CI to build on ubuntu only and test on both --- .github/workflows/devcontainer.yml | 66 +++++++++++++++++++----------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index ca1304bf8..3eab92f3e 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -12,23 +12,40 @@ on: jobs: - build-and-push: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Build Dev Container Image + uses: devcontainers/ci@v0.2 + with: + subFolder: .github + imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }} + push: never + + test: + needs: build strategy: fail-fast: false matrix: - os: - - [ubuntu-latest] - - [ macos-13 ] + os: [ubuntu-latest, macos-13] runs-on: ${{ matrix.os }} steps: - - - name: Checkout - id: checkout - uses: actions/checkout@v1 - - - name: Setup docker (missing on MacOS) + - name: Setup docker (macOS only) if: runner.os == 'macos' run: | brew install docker docker-buildx @@ -37,27 +54,26 @@ jobs: while ! docker info >/dev/null 2>&1; do sleep 1; done mkdir -p ~/.docker/cli-plugins ln -sfn $(which docker-buildx) ~/.docker/cli-plugins/docker-buildx - + + - name: Test Container Image + run: | + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" + + push: + needs: [build, test] + runs-on: ubuntu-latest + if: ${{ always() && success() }} + + steps: - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Build Dev Container Image - uses: devcontainers/ci@v0.2 - with: - subFolder: .github - imageName: ghcr.io/${{ github.repository }} - cacheFrom: ghcr.io/${{ github.repository }} - push: never - - - name: Test Container Image - run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - name: Push Container Image - if: ${{ success() }} run: | - docker push ghcr.io/${{ github.repository }} \ No newline at end of file + docker push ghcr.io/${{ github.repository }} + + From 07590df296762487364b05cdcdb4408e33a00856 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:12:14 +0700 Subject: [PATCH 053/113] pushing test image before runningn tests --- .github/workflows/devcontainer.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 3eab92f3e..6f3030b3d 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -31,9 +31,9 @@ jobs: uses: devcontainers/ci@v0.2 with: subFolder: .github - imageName: ghcr.io/${{ github.repository }} - cacheFrom: ghcr.io/${{ github.repository }} - push: never + imageName: ghcr.io/${{ github.repository }}:test + cacheFrom: ghcr.io/${{ github.repository }}:test + push: always test: needs: build @@ -57,7 +57,7 @@ jobs: - name: Test Container Image run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" + docker run --rm ghcr.io/${{ github.repository }}:test /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" push: needs: [build, test] @@ -74,6 +74,6 @@ jobs: - name: Push Container Image run: | - docker push ghcr.io/${{ github.repository }} + docker push ghcr.io/${{ github.repository }}:latest From 9629a2e5d577b8b33ac6377cd0e0046350a76829 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:21:00 +0700 Subject: [PATCH 054/113] adding updated build action --- .github/workflows/devcontainer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 6f3030b3d..ce80cf933 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -28,7 +28,7 @@ jobs: password: ${{ secrets.REGISTRY_TOKEN }} - name: Build Dev Container Image - uses: devcontainers/ci@v0.2 + uses: devcontainers/ci@v0.3 with: subFolder: .github imageName: ghcr.io/${{ github.repository }}:test From fc27e5a7ae43b8677a04361a0a3ab4f9db68d6d6 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:28:15 +0700 Subject: [PATCH 055/113] removing cache --- .github/workflows/devcontainer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index ce80cf933..416661803 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -32,7 +32,6 @@ jobs: with: subFolder: .github imageName: ghcr.io/${{ github.repository }}:test - cacheFrom: ghcr.io/${{ github.repository }}:test push: always test: From a0ba73a84bee6416c5ddf34405350589d0497957 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:30:48 +0700 Subject: [PATCH 056/113] removing tagging --- .github/workflows/devcontainer.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 416661803..1f03a34d0 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -31,7 +31,8 @@ jobs: uses: devcontainers/ci@v0.3 with: subFolder: .github - imageName: ghcr.io/${{ github.repository }}:test + cacheFrom: ghcr.io/${{ github.repository }} + imageName: ghcr.io/${{ github.repository }} push: always test: @@ -56,7 +57,7 @@ jobs: - name: Test Container Image run: | - docker run --rm ghcr.io/${{ github.repository }}:test /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" + docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" push: needs: [build, test] @@ -73,6 +74,6 @@ jobs: - name: Push Container Image run: | - docker push ghcr.io/${{ github.repository }}:latest + docker push ghcr.io/${{ github.repository }} From e1c40540813f0c799ed8327862a005a416f26e5a Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:39:58 +0700 Subject: [PATCH 057/113] adding no cache to build --- .github/workflows/devcontainer.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 1f03a34d0..65d2fff23 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -34,13 +34,14 @@ jobs: cacheFrom: ghcr.io/${{ github.repository }} imageName: ghcr.io/${{ github.repository }} push: always + noCache: true test: needs: build strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-13] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} From bc4e8fbd5cb171d9e89a97f14cd87303ac335016 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 13 Nov 2024 17:44:13 +0700 Subject: [PATCH 058/113] no need to push again after checking --- .github/workflows/devcontainer.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 65d2fff23..8a3cb8372 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -60,21 +60,4 @@ jobs: run: | docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - push: - needs: [build, test] - runs-on: ubuntu-latest - if: ${{ always() && success() }} - - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Push Container Image - run: | - docker push ghcr.io/${{ github.repository }} - From 4d98000cf510aaf1dedfbd39e004478b978f5e39 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 6 Dec 2024 13:07:59 +0100 Subject: [PATCH 059/113] Update devcontainer.yml --- .github/workflows/devcontainer.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 8a3cb8372..4d1e86528 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -35,29 +35,3 @@ jobs: imageName: ghcr.io/${{ github.repository }} push: always noCache: true - - test: - needs: build - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - - runs-on: ${{ matrix.os }} - - steps: - - name: Setup docker (macOS only) - if: runner.os == 'macos' - run: | - brew install docker docker-buildx - brew install colima - colima start - while ! docker info >/dev/null 2>&1; do sleep 1; done - mkdir -p ~/.docker/cli-plugins - ln -sfn $(which docker-buildx) ~/.docker/cli-plugins/docker-buildx - - - name: Test Container Image - run: | - docker run --rm ghcr.io/${{ github.repository }} /bin/bash -c "git clone -b devContainers https://github.com/drifter089/pypsa-earth.git && cd pypsa-earth && bash .github/.devcontainer/test.sh" - - From c610ddc8d875c4c5741723a6ae22dc0e0a090c65 Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Fri, 6 Dec 2024 19:14:24 +0700 Subject: [PATCH 060/113] Update Dockerfile using linux-pinned --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2b171658..57e72d6a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /pypsa-earth COPY ./envs ./temp -RUN conda env create -n pypsa-earth -f temp/environment.yaml +RUN conda env create -n pypsa-earth -f temp/linux-pinned.yaml RUN conda init bash @@ -27,7 +27,5 @@ RUN rm -r temp RUN conda clean -afy && \ rm -rf /tmp/* -ENV LD_PRELOAD=/opt/conda/envs/pypsa-earth/lib/python3.10/site-packages/sklearn/utils/../../../../libgomp.so.1 - CMD ["bash"] From 9c983caeb330a79c6692da2b97ed3b02a9f19f11 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 6 Dec 2024 13:26:56 +0100 Subject: [PATCH 061/113] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 57e72d6a4..a480b3780 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ RUN conda update -n base conda RUN conda install -n base conda-libmamba-solver RUN conda config --set solver libmamba -RUN apt-get update && apt-get install -y bash git +RUN apt-get update && apt-get install -y bash git && apt-get install gcc -y WORKDIR /pypsa-earth From 7f93dd07b6f85048a1bad3952cd2a96004e72ba7 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 10 Dec 2024 19:46:59 +0100 Subject: [PATCH 062/113] Put country-specific data into resources folder --- Snakefile | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/Snakefile b/Snakefile index 767833df7..086a4421a 100644 --- a/Snakefile +++ b/Snakefile @@ -1004,7 +1004,7 @@ rule prepare_ports: params: custom_export=config["custom_data"]["export_ports"], output: - ports="data/ports.csv", # TODO move from data to resources + ports="resources/" + SECDIR + "ports.csv", export_ports="resources/" + SECDIR + "export_ports.csv", script: "scripts/prepare_ports.py" @@ -1014,14 +1014,14 @@ rule prepare_airports: params: airport_sizing_factor=config["sector"]["airport_sizing_factor"], output: - ports="data/airports.csv", # TODO move from data to resources + ports="resources/" + SECDIR + "airports.csv", script: "scripts/prepare_airports.py" rule prepare_urban_percent: output: - urban_percent="data/urban_percent.csv", # TODO move from data to resources + urban_percent="resources/" + SECDIR + "urban_percent.csv", script: "scripts/prepare_urban_percent.py" @@ -1094,9 +1094,11 @@ rule prepare_sector_network: industrial_demand="resources/" + SECDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", - energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", - airports="data/airports.csv", - ports="data/ports.csv", + energy_totals="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", + airports="resources/" + SECDIR + "airports.csv", + ports="resources/" + SECDIR + "ports.csv", heat_demand="resources/" + SECDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", @@ -1200,7 +1202,9 @@ rule override_respot: }, overrides="data/override_component_attrs", network="networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", + energy_totals="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", output: RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", @@ -1211,7 +1215,9 @@ rule override_respot: rule prepare_transport_data: input: network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", - energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", + energy_totals_name="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", transport_name="resources/" + SECDIR + "transport_data.csv", @@ -1295,7 +1301,9 @@ rule build_cop_profiles: rule prepare_heat_data: input: network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", - energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", + energy_totals_name="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", clustered_pop_layout="resources/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", @@ -1348,7 +1356,7 @@ rule build_base_energy_totals: input: unsd_paths="data/demand/unsd/paths/Energy_Statistics_Database.xlsx", output: - energy_totals_base="data/energy_totals_base.csv", + energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv", script: "scripts/build_base_energy_totals.py" @@ -1359,13 +1367,15 @@ rule prepare_energy_totals: base_year=config["demand_data"]["base_year"], sector_options=config["sector"], input: - unsd_paths="data/energy_totals_base.csv", + unsd_paths="resources/" + SECDIR + "energy_totals_base.csv", efficiency_gains_cagr="data/demand/efficiency_gains_cagr.csv", growth_factors_cagr="data/demand/growth_factors_cagr.csv", district_heating="data/demand/district_heating.csv", fuel_shares="data/demand/fuel_shares.csv", output: - energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", + energy_totals="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", script: "scripts/prepare_energy_totals.py" @@ -1419,7 +1429,7 @@ rule build_population_layouts: planning_horizons=config["scenario"]["planning_horizons"][0], input: nuts3_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson", - urban_percent="data/urban_percent.csv", + urban_percent="resources/" + SECDIR + "urban_percent.csv", cutout="cutouts/" + CDIR + [c["cutout"] for _, c in config["renewable"].items()][0] @@ -1866,7 +1876,7 @@ rule build_base_industry_totals: #default data input: #industrial_production_per_country="data/industrial_production_per_country.csv", #unsd_path="data/demand/unsd/data/", - energy_totals_base="data/energy_totals_base.csv", + energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv", transactions_path="data/unsd_transactions.csv", output: base_industry_totals="resources/" From 26eafdb54e0e53b1b5f08cf3600e4ed37a677209 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 10 Dec 2024 19:47:33 +0100 Subject: [PATCH 063/113] Replace a benchmark path --- scripts/build_base_industry_totals.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_base_industry_totals.py b/scripts/build_base_industry_totals.py index eef58f10a..46736c61c 100644 --- a/scripts/build_base_industry_totals.py +++ b/scripts/build_base_industry_totals.py @@ -117,8 +117,9 @@ def create_industry_base_totals(df): renaming_dit = transaction.set_index("Transaction")["clean_name"].to_dict() clean_industry_list = list(transaction.clean_name.unique()) + # TODO Check with the sector-coupled modelling experts unsd_path = ( - os.path.dirname(snakemake.input["energy_totals_base"]) + "/demand/unsd/data/" + os.path.dirname(snakemake.input["transactions_path"]) + "/demand/unsd/data/" ) # Get the files from the path provided in the OP From 68bb45d0fc23bfd1c3ac162ffb8547035e90ea13 Mon Sep 17 00:00:00 2001 From: drifter089 Date: Wed, 11 Dec 2024 16:59:29 +0700 Subject: [PATCH 064/113] pre-commit bot run --- .devcontainer/devcontainer.json | 2 +- .devcontainer/setup.sh | 4 ++-- .devcontainer/welcome-message.txt | 2 +- .github/.devcontainer/test.sh | 2 +- .github/workflows/devcontainer.yml | 10 +++++----- Docker.MD | 2 +- Dockerfile | 1 - doc/docker_containers.rst | 2 +- 8 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 81e1605d9..fe8c094b5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ -// For format details, see https://aka.ms/devcontainer.json. +// For format details, see https://aka.ms/devcontainer.json. { "name": "pypsa earth dev", "image": "ghcr.io/drifter089/pypsa-earth:latest", diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index e2fc1a3e8..deec01786 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,6 +1,6 @@ #!/bin/sh -# mv ../pypsa-eur/* ./ +# mv ../pypsa-eur/* ./ # echo 'solved tutorial files are copied to workspace' -cat .devcontainer/welcome-message.txt \ No newline at end of file +cat .devcontainer/welcome-message.txt diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt index 9fe33b7d6..081694bdf 100644 --- a/.devcontainer/welcome-message.txt +++ b/.devcontainer/welcome-message.txt @@ -18,4 +18,4 @@ We’re excited to have you here! This setup allows you to contribute to PyPSA-E 🚀 Start Exploring and Happy Coding! -Don’t hesitate to reach out if you need help—our community is here to support you. \ No newline at end of file +Don’t hesitate to reach out if you need help—our community is here to support you. diff --git a/.github/.devcontainer/test.sh b/.github/.devcontainer/test.sh index c4ac3f290..78556dbcd 100644 --- a/.github/.devcontainer/test.sh +++ b/.github/.devcontainer/test.sh @@ -2,4 +2,4 @@ cp config.tutorial.yaml config.yaml -snakemake -j 1 solve_all_networks \ No newline at end of file +snakemake -j 1 solve_all_networks diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 4d1e86528..222048c33 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -4,13 +4,13 @@ on: workflow_dispatch: push: branches: - - "main" + - "main" tags: - - "v*.*.*" + - "v*.*.*" pull_request: - branches: [main] + branches: [main] + - jobs: build: runs-on: ubuntu-latest @@ -33,5 +33,5 @@ jobs: subFolder: .github cacheFrom: ghcr.io/${{ github.repository }} imageName: ghcr.io/${{ github.repository }} - push: always + push: always noCache: true diff --git a/Docker.MD b/Docker.MD index 45e783e77..e7adbc01c 100644 --- a/Docker.MD +++ b/Docker.MD @@ -46,4 +46,4 @@ Once these steps are completed, your development environment will be ready, and You are now all set up! You can use the development environment to explore PyPSA-Earth and make modifications as needed within the Docker container. -You can start running the tutorial [here](https://pypsa-earth.readthedocs.io/en/latest/short_tutorial.html) \ No newline at end of file +You can start running the tutorial [here](https://pypsa-earth.readthedocs.io/en/latest/short_tutorial.html) diff --git a/Dockerfile b/Dockerfile index a480b3780..bb831e973 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,3 @@ RUN conda clean -afy && \ rm -rf /tmp/* CMD ["bash"] - diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index bc0ea71af..fdc561f83 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -45,4 +45,4 @@ Steps: The environment will be ready for use. You can now run PyPSA-Earth in the container. - \ No newline at end of file + From dca591b2cdfd4b3f18eebdb192730d708f64c6d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:03:11 +0000 Subject: [PATCH 065/113] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/docker_containers.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index fdc561f83..df979711b 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -44,5 +44,3 @@ Steps: * Wait for the container to build and open the repository in the container. The environment will be ready for use. You can now run PyPSA-Earth in the container. - - From 6effd98172eafdd8f4d7d03b3d8e1c97321d2e0f Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:30:36 +0700 Subject: [PATCH 066/113] Delete .github/.devcontainer/test.sh --- .github/.devcontainer/test.sh | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .github/.devcontainer/test.sh diff --git a/.github/.devcontainer/test.sh b/.github/.devcontainer/test.sh deleted file mode 100644 index 78556dbcd..000000000 --- a/.github/.devcontainer/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cp config.tutorial.yaml config.yaml - -snakemake -j 1 solve_all_networks From d793625e5afa112c526087062c1e5f924a8405ab Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:31:37 +0700 Subject: [PATCH 067/113] Update devcontainer.yml --- .github/workflows/devcontainer.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 222048c33..5148df2be 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -3,13 +3,8 @@ name: Dev Container Build and Push Image on: workflow_dispatch: push: - branches: - - "main" - tags: - - "v*.*.*" - pull_request: - branches: [main] - + paths: + - envs/linux-pinned.yaml jobs: build: From f5a957eb70d0ec53277939f5243a36d62ad5a9d5 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 11 Dec 2024 15:39:13 +0100 Subject: [PATCH 068/113] Replace path hardcoding in script --- scripts/prepare_energy_totals.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py index 708420b27..9e473de5e 100644 --- a/scripts/prepare_energy_totals.py +++ b/scripts/prepare_energy_totals.py @@ -43,8 +43,8 @@ def calculate_end_values(df): snakemake = mock_snakemake( "prepare_energy_totals", simpl="", - clusters=32, - demand="EG", + clusters=10, + demand="AB", planning_horizons=2030, ) @@ -53,9 +53,7 @@ def calculate_end_values(df): investment_year = int(snakemake.wildcards.planning_horizons) demand_sc = snakemake.wildcards.demand # loading the demand scenrario wildcard - base_energy_totals = read_csv_nafix( - os.path.join(BASE_DIR, "data/energy_totals_base.csv"), index_col=0 - ) + base_energy_totals = read_csv_nafix(snakemake.input.unsd_paths, index_col=0) growth_factors_cagr = read_csv_nafix( snakemake.input.growth_factors_cagr, index_col=0 ) From 4d659a3299ea20c572738445a18119040da0f89a Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Fri, 13 Dec 2024 13:08:09 +0100 Subject: [PATCH 069/113] fix precommit issues --- .devcontainer/devcontainer.json | 3 +++ .devcontainer/setup.sh | 3 +++ .devcontainer/welcome-message.txt | 4 +++- Docker.MD | 5 +++++ Dockerfile | 3 +++ doc/docker_containers.rst | 3 +++ 6 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fe8c094b5..02a0c928d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +// +// SPDX-License-Identifier: CC-BY-4.0 // For format details, see https://aka.ms/devcontainer.json. { "name": "pypsa earth dev", diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index deec01786..f57177eea 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 #!/bin/sh # mv ../pypsa-eur/* ./ diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt index 081694bdf..afcec3035 100644 --- a/.devcontainer/welcome-message.txt +++ b/.devcontainer/welcome-message.txt @@ -1,4 +1,6 @@ - +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 diff --git a/Docker.MD b/Docker.MD index e7adbc01c..c2db2aaa7 100644 --- a/Docker.MD +++ b/Docker.MD @@ -1,3 +1,8 @@ + # PyPSA-Earth Development Environment Setup with Docker This guide provides an alternative way to set up a development environment for **PyPSA-Earth** using Docker containers. This method is particularly useful for users who are not familiar with programming or Python, or who prefer not to install Python directly on their local machine. Using Docker simplifies setup by creating a self-contained environment for PyPSA-Earth. diff --git a/Dockerfile b/Dockerfile index bb831e973..1873210a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC-BY-4.0 FROM condaforge/mambaforge RUN conda update -n base conda diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index df979711b..bc55423b7 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -1,3 +1,6 @@ +.. SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +.. +.. SPDX-License-Identifier: CC-BY-4.0 .. _docker_containers: From 1420450403b6101282960e39672d1b5edbca378c Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Sat, 14 Dec 2024 19:58:22 +0700 Subject: [PATCH 070/113] trigger and image name --- .github/workflows/devcontainer.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 5148df2be..12288a6b1 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -3,6 +3,7 @@ name: Dev Container Build and Push Image on: workflow_dispatch: push: + branches: [main] paths: - envs/linux-pinned.yaml @@ -26,7 +27,7 @@ jobs: uses: devcontainers/ci@v0.3 with: subFolder: .github - cacheFrom: ghcr.io/${{ github.repository }} - imageName: ghcr.io/${{ github.repository }} + cacheFrom: ghcr.io/${{ github.repository }}-dev-env + imageName: ghcr.io/${{ github.repository }}-dev-env push: always - noCache: true + # noCache: true From c8654cfeaf146c90b96961e31bfd41f9f235bb79 Mon Sep 17 00:00:00 2001 From: Akshat Mittal <93286254+drifter089@users.noreply.github.com> Date: Sat, 14 Dec 2024 20:02:16 +0700 Subject: [PATCH 071/113] Update setup.sh --- .devcontainer/setup.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index f57177eea..a3ef9d7bb 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -3,7 +3,4 @@ # SPDX-License-Identifier: CC-BY-4.0 #!/bin/sh -# mv ../pypsa-eur/* ./ -# echo 'solved tutorial files are copied to workspace' - cat .devcontainer/welcome-message.txt From e03d724948c8ccadebd8583add5e9ae5614b9e03 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Sat, 21 Dec 2024 13:40:06 +0100 Subject: [PATCH 072/113] restore BJ to country selection --- config.tutorial.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 3ad688678..1c5bf7552 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -6,7 +6,7 @@ version: 0.4.2 tutorial: true -countries: ["NG"] +countries: ["NG", "BJ"] enable: retrieve_databundle: true From 5dfae1c95ed4d17541346167469a8c18e4a22e95 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Sat, 21 Dec 2024 19:43:35 +0100 Subject: [PATCH 073/113] Add a workflow to manually update pinned environments for PR (#1250) * Add a workflow to manually update pinned environments for PR * Add a run condition to update the pinned envs for branches * Remove a duplicating workflow --- .github/workflows/update-pinned-env.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update-pinned-env.yml b/.github/workflows/update-pinned-env.yml index 6f35831c6..4b0794f2f 100644 --- a/.github/workflows/update-pinned-env.yml +++ b/.github/workflows/update-pinned-env.yml @@ -12,7 +12,8 @@ on: jobs: update-pinned-environment: - if: ${{ github.ref == 'refs/heads/main' }} + # the update is always needed for env changes in main and sometimes for other branches + if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }} name: Update pinned envs runs-on: ${{ matrix.os }}-latest strategy: From b1ff2985d58b1ea880b4417110677071f7642b15 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Sun, 22 Dec 2024 00:00:02 +0100 Subject: [PATCH 074/113] Update documenter frequency (#1252) * Update documenter frequency * Ensure execution on main only --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4e9fe2e8a..9ebd17f4d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,11 +1,13 @@ on: schedule: - - cron: "0 5 * * 0" + - cron: "0 5 1 * *" + workflow_dispatch: jobs: contrib-readme-job: runs-on: ubuntu-latest name: A job to automate contrib in readme + if: ${{ github.repository_owner == 'pypsa-meets-earth' && github.ref == 'refs/heads/main'}} steps: - name: Contribute List uses: akhilmhdh/contributors-readme-action@v2.3.10 From 88d4cab9ba65c0924b74218ef23043fdf7218815 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 22 Dec 2024 00:32:23 +0100 Subject: [PATCH 075/113] Update links and descriptions --- doc/users_list.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index 1d0a4cbda..44e425440 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -7,14 +7,18 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Energy transition pathways for Nigeria (Research paper) +Power system modeling for Korea to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) -EDF project +Development a model for the Pakistan Power Systemb (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. -Implications of the national decarbonisaiton goals for power system of Kazakhstan (project of Open Energy Transition + Agora Energiewende) +The modeling study has been done as a part of Master Work in Mehran University of Engineering and Technology as presented in the `post `_ -Possible implementation of energy transition in Saudi Arabia (MSc Thesis in Edinburgh University) +Investigation of implications of the national decarbonisaiton goals for power system of Kazakhstan. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going beyond the official mid-term goal of 15% renewable energy share by 2030. -Power system modeling for Korea to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) +Exploring more ambitious energy transition pathways for Kazakhstan has been is subject of a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. + +PyPSA-Earth has been used in the Multi-Country Electricity Transition Potential and Challenges Project (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in individual countries for power system planning in deep decarbonization scenarios + +Investigation of the energy transition pathways for Nigeria published in a `research paper `_ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behaviour and one representing a decarbonized 2060 scenario. -Pakistan +Possible implementation of energy transition in Saudi Arabia published as `MSc dissertation `_ in Edinburgh University. The outputs of the work are available in the `repository `_. From 3ebf4b767bc4362585dbd5967852c62f3a3b0f20 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 22 Dec 2024 16:59:47 +0100 Subject: [PATCH 076/113] Fix control-related formatting --- doc/users_list.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index 44e425440..8eda2ae1c 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -2,12 +2,11 @@ .. .. SPDX-License-Identifier: CC-BY-4.0 -.. PyPSA meets Earth documentation master file, created by - sphinx-quickstart on Sat May 15 22:52:54 2021. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. _users_list: -Power system modeling for Korea to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) +########################################## +PyPSA-Earth Applications +########################################## Development a model for the Pakistan Power Systemb (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. From c1187186cfede5d7f52748e3c32cd52512ec426c Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 22 Dec 2024 17:02:15 +0100 Subject: [PATCH 077/113] Improve text formatting --- doc/users_list.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index 8eda2ae1c..3a561fdcc 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -8,16 +8,14 @@ PyPSA-Earth Applications ########################################## -Development a model for the Pakistan Power Systemb (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. +Power system modeling for **Korea** to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) -The modeling study has been done as a part of Master Work in Mehran University of Engineering and Technology as presented in the `post `_ +Development a model for the **Pakistan** Power System (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. The modeling study has been done as a part of Master Work in Mehran University of Engineering and Technology as presented in the `post `_ -Investigation of implications of the national decarbonisaiton goals for power system of Kazakhstan. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going beyond the official mid-term goal of 15% renewable energy share by 2030. +Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going beyond the official mid-term goal of 15% renewable energy share by 2030. Exploring more ambitious energy transition pathways for Kazakhstan has been is subject of a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. -Exploring more ambitious energy transition pathways for Kazakhstan has been is subject of a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. +**Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in China, India, Colombia, Chile, Kazakhstan, Bangladesh, Vietnam and Thailand for power system planning in deep decarbonization scenarios. -PyPSA-Earth has been used in the Multi-Country Electricity Transition Potential and Challenges Project (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in individual countries for power system planning in deep decarbonization scenarios +Investigation of the energy transition pathways for **Nigeria** published in a `research paper `_ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behaviour and one representing a decarbonized 2060 scenario. -Investigation of the energy transition pathways for Nigeria published in a `research paper `_ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behaviour and one representing a decarbonized 2060 scenario. - -Possible implementation of energy transition in Saudi Arabia published as `MSc dissertation `_ in Edinburgh University. The outputs of the work are available in the `repository `_. +Possible implementation of energy transition in **Saudi Arabia** published as `MSc dissertation `_ in Edinburgh University. The outputs of the work are available in the `repository `_. From 67e8f55a6d9ae38d5f8c069f0b3c1ef6ca86954c Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 22 Dec 2024 17:04:45 +0100 Subject: [PATCH 078/113] Improve header --- doc/users_list.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index 3a561fdcc..936fad943 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -5,7 +5,7 @@ .. _users_list: ########################################## -PyPSA-Earth Applications +PyPSA-Earth Usage Examples ########################################## Power system modeling for **Korea** to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) From 9c5dea9102761df4ff961c9ccf002171c94f3bbd Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 22 Dec 2024 17:15:16 +0100 Subject: [PATCH 079/113] Minor improvements --- doc/users_list.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index 936fad943..e66296beb 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -5,17 +5,17 @@ .. _users_list: ########################################## -PyPSA-Earth Usage Examples +Usage of PyPSA-Earth ########################################## -Power system modeling for **Korea** to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) +* Power system modeling for **Korea** to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) -Development a model for the **Pakistan** Power System (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. The modeling study has been done as a part of Master Work in Mehran University of Engineering and Technology as presented in the `post `_ +* Development a model for the **Pakistan** Power System (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. The modeling study has been done as a part of Master Work in Mehran University of Engineering and Technology as presented in the `post `_ -Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going beyond the official mid-term goal of 15% renewable energy share by 2030. Exploring more ambitious energy transition pathways for Kazakhstan has been is subject of a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. +* Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going beyond the official mid-term goal of 15% renewable energy share by 2030. Exploring more ambitious energy transition pathways for Kazakhstan has been is subject of a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. -**Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in China, India, Colombia, Chile, Kazakhstan, Bangladesh, Vietnam and Thailand for power system planning in deep decarbonization scenarios. +* **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in China, India, Colombia, Chile, Kazakhstan, Bangladesh, Vietnam and Thailand for power system planning in deep decarbonization scenarios. -Investigation of the energy transition pathways for **Nigeria** published in a `research paper `_ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behaviour and one representing a decarbonized 2060 scenario. +* Investigation of the energy transition pathways for **Nigeria** published in a `research paper `_ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behaviour and one representing a decarbonized 2060 scenario. -Possible implementation of energy transition in **Saudi Arabia** published as `MSc dissertation `_ in Edinburgh University. The outputs of the work are available in the `repository `_. +* Possible implementation of energy transition in **Saudi Arabia** published as `MSc dissertation `_ in Edinburgh University. The outputs of the work are available in the `repository `_. From 6891d331f1f4388f9ff1fa124049001880ff9c51 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 22 Dec 2024 17:22:31 +0100 Subject: [PATCH 080/113] Add some details to a description of Saudi Arabia model --- doc/users_list.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index e66296beb..d64d5e295 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -14,8 +14,8 @@ Usage of PyPSA-Earth * Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going beyond the official mid-term goal of 15% renewable energy share by 2030. Exploring more ambitious energy transition pathways for Kazakhstan has been is subject of a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. -* **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in China, India, Colombia, Chile, Kazakhstan, Bangladesh, Vietnam and Thailand for power system planning in deep decarbonization scenarios. +* **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in **China**, **India**, **Colombia**, **Chile**, **Kazakhstan**, **Bangladesh**, **Vietnam** and **Thailand** for power system planning in deep decarbonization scenarios. * Investigation of the energy transition pathways for **Nigeria** published in a `research paper `_ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behaviour and one representing a decarbonized 2060 scenario. -* Possible implementation of energy transition in **Saudi Arabia** published as `MSc dissertation `_ in Edinburgh University. The outputs of the work are available in the `repository `_. +* Modeling of the power sector in **Saudi Arabia** published as `MSc dissertation `_ in Edinburgh University. This has been a first attempt to predict the energy system for Saudi Arabia in 2060, with net zero emissions. The models in this study include a base model for validation, a model for 2030, to compare to the country’s Vision 2030 and as a transition phase, and two models for 2060 net zero, one is fully renewable and the other utilizing carbon capture and storage technology. The outputs of the work are available in the `repository `_. From 4e8a738115e559151e26eb360001229d677c89c8 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 22 Dec 2024 17:34:28 +0100 Subject: [PATCH 081/113] Fix a description if Kazakhstan project --- doc/users_list.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index d64d5e295..34f8d1b00 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -12,7 +12,7 @@ Usage of PyPSA-Earth * Development a model for the **Pakistan** Power System (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. The modeling study has been done as a part of Master Work in Mehran University of Engineering and Technology as presented in the `post `_ -* Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going beyond the official mid-term goal of 15% renewable energy share by 2030. Exploring more ambitious energy transition pathways for Kazakhstan has been is subject of a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. +* Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going even beyond the official mid-term goal of 15% renewable energy share by 2030 in a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. * **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in **China**, **India**, **Colombia**, **Chile**, **Kazakhstan**, **Bangladesh**, **Vietnam** and **Thailand** for power system planning in deep decarbonization scenarios. From c5ac26d5684f44a676755de85c0819493fb83579 Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 23 Dec 2024 15:19:10 +0100 Subject: [PATCH 082/113] Fix CI bage --- doc/index.rst | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index fe7fb88e1..3385411be 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -14,8 +14,8 @@ Welcome to the PyPSA-Earth documentation! .. image:: https://img.shields.io/github/v/release/pypsa-meets-earth/pypsa-earth?include_prereleases :alt: GitHub release (latest by date including pre-releases) -.. image:: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci.yml/badge.svg - :target: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci.yml +.. image:: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/test.yml/badge.svg + :target: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/test.yml :alt: CI .. image:: https://readthedocs.org/projects/pypsa-earth/badge/?version=latest @@ -178,23 +178,26 @@ Documentation monte_carlo -**Applications** +**Support and Contributing** -* :doc:`users_list` +* :doc:`release_notes` +* :doc:`how_to_contribute` +* :doc:`software_hints` +* :doc:`learning_materials` .. toctree:: :hidden: :maxdepth: 2 - :caption: Applications + :caption: Support - users_list + release_notes + how_to_contribute + software_hints + learning_materials -**Help and References** +**References** -* :doc:`release_notes` -* :doc:`how_to_contribute` -* :doc:`software_hints` -* :doc:`learning_materials` +* :doc:`users_list` * :doc:`project_structure_and_credits` * :doc:`talks_and_papers` @@ -203,9 +206,6 @@ Documentation :maxdepth: 2 :caption: Project Info - release_notes - how_to_contribute - software_hints - learning_materials + users_list project_structure_and_credits talks_and_papers From 6fb47f55a6ef1bf57a7853d91ec1999743a10c10 Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 23 Dec 2024 15:20:27 +0100 Subject: [PATCH 083/113] Add an intro --- doc/users_list.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index 34f8d1b00..2c1dbc9a4 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -7,12 +7,13 @@ ########################################## Usage of PyPSA-Earth ########################################## +The list below contains some the applications of PyPSA-Earth for educational, research and industrial projects and is likely to be incomplete. If you want to add your contribution into the list, feel free to open a PR to `PyPSA Earth repo `__ or flag your interest via `PyPSA meets Earth Discord server `__. -* Power system modeling for **Korea** to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `_ at `Korea University `_) +* Power system modeling for **Korea** to support development of the national energy strategy, considering ESS Capacity expansion, LMP & Cost analysis according to ESS combination, and coal phase-out scenario (`Energy Innovation Lab `__ at `Korea University `__) -* Development a model for the **Pakistan** Power System (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. The modeling study has been done as a part of Master Work in Mehran University of Engineering and Technology as presented in the `post `_ +* Development a model for the **Pakistan** Power System (Soon available on GitHub in PyPSA-Earth's repository) cross-referenced and validated with national reports. The model has been used to develop the Net-Zero scenario by 2050 utilizing Pakistan's eligible Wind and Solar potential. The modeling study has been done as a part of Master Work in Mehran University of Engineering and Technology as presented in the `post `__ -* Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going even beyond the official mid-term goal of 15% renewable energy share by 2030 in a joint project developed by Open Energy Transition and Agora Energiewende which `report `_ has been published openly. +* Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going even beyond the official mid-term goal of 15% renewable energy share by 2030 in a joint project developed by Open Energy Transition and Agora Energiewende which `report `__ has been published openly. * **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in **China**, **India**, **Colombia**, **Chile**, **Kazakhstan**, **Bangladesh**, **Vietnam** and **Thailand** for power system planning in deep decarbonization scenarios. From 838ad034ae01091f4fad5f36d97ed69b87b2e16f Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 23 Dec 2024 15:22:35 +0100 Subject: [PATCH 084/113] Add a link to the country-wise paper --- doc/users_list.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index 2c1dbc9a4..a48279dc6 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -15,8 +15,10 @@ The list below contains some the applications of PyPSA-Earth for educational, re * Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going even beyond the official mid-term goal of 15% renewable energy share by 2030 in a joint project developed by Open Energy Transition and Agora Energiewende which `report `__ has been published openly. -* **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `_. The goal was to analyze barriers to decarbonization in **China**, **India**, **Colombia**, **Chile**, **Kazakhstan**, **Bangladesh**, **Vietnam** and **Thailand** for power system planning in deep decarbonization scenarios. +* Country-wise validation of PyPSA-Earth methodology to extract modeling-ready data for power system modeling considering **any country on the Earth** published as a `conference paper `__. Results confirm that PyPSA-Earth now covers approximately 99% of the world's population and can generate high-resolution datasets for national energy studies. This work lays the foundations for future endeavors in flexible energy modeling of arbitrarily large regions. The supplementary data for fully reproducible 193+ power systems images created with PyPSA-Earth for all United Nations countries are available in ` zenodo repozitory `__. -* Investigation of the energy transition pathways for **Nigeria** published in a `research paper `_ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behaviour and one representing a decarbonized 2060 scenario. +* **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `__. The goal was to analyze barriers to decarbonization in **China**, **India**, **Colombia**, **Chile**, **Kazakhstan**, **Bangladesh**, **Vietnam** and **Thailand** for power system planning in deep decarbonization scenarios. -* Modeling of the power sector in **Saudi Arabia** published as `MSc dissertation `_ in Edinburgh University. This has been a first attempt to predict the energy system for Saudi Arabia in 2060, with net zero emissions. The models in this study include a base model for validation, a model for 2030, to compare to the country’s Vision 2030 and as a transition phase, and two models for 2060 net zero, one is fully renewable and the other utilizing carbon capture and storage technology. The outputs of the work are available in the `repository `_. +* Investigation of the energy transition pathways for **Nigeria** published in a `research paper `__ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behavior and one representing a decarbonized 2060 scenario. + +* Modeling of the power sector in **Saudi Arabia** published as `MSc dissertation `__ in Edinburgh University. This has been a first attempt to predict the energy system for Saudi Arabia in 2060, with net zero emissions. The models in this study include a base model for validation, a model for 2030, to compare to the country’s Vision 2030 and as a transition phase, and two models for 2060 net zero, one is fully renewable and the other utilizing carbon capture and storage technology. The outputs of the work are available in the `repository `__. From f9c9a74d28a80083ac4c869854fbead1c99089be Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 23 Dec 2024 15:30:11 +0100 Subject: [PATCH 085/113] Minor fixes --- doc/users_list.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/users_list.rst b/doc/users_list.rst index a48279dc6..118f193f6 100644 --- a/doc/users_list.rst +++ b/doc/users_list.rst @@ -15,10 +15,10 @@ The list below contains some the applications of PyPSA-Earth for educational, re * Investigation of implications of the national decarbonisaiton goals for power system of **Kazakhstan**. Traditionally, Kazakhstan’s power sector is using a lot of coal, while the national plans introduce a significant increase in solar and wind generation during the next decades. Model Kazakhstan power system has been used to investigate economic feasibility of going even beyond the official mid-term goal of 15% renewable energy share by 2030 in a joint project developed by Open Energy Transition and Agora Energiewende which `report `__ has been published openly. -* Country-wise validation of PyPSA-Earth methodology to extract modeling-ready data for power system modeling considering **any country on the Earth** published as a `conference paper `__. Results confirm that PyPSA-Earth now covers approximately 99% of the world's population and can generate high-resolution datasets for national energy studies. This work lays the foundations for future endeavors in flexible energy modeling of arbitrarily large regions. The supplementary data for fully reproducible 193+ power systems images created with PyPSA-Earth for all United Nations countries are available in ` zenodo repozitory `__. +* Country-wise validation of PyPSA-Earth methodology to extract modeling-ready data for power system modeling considering **any country on the Earth** published as a `conference paper `__. Results confirm that PyPSA-Earth now covers approximately 99% of the world's population and can generate high-resolution datasets for national energy studies. The supplementary data for fully reproducible 193+ power systems images created with PyPSA-Earth for all United Nations countries are available in `zenodo repozitory `__. -* **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `__. The goal was to analyze barriers to decarbonization in **China**, **India**, **Colombia**, **Chile**, **Kazakhstan**, **Bangladesh**, **Vietnam** and **Thailand** for power system planning in deep decarbonization scenarios. +* **Multi-Country Electricity Transition Potential and Challenges Project** (MCET) of `Environmental Defence Fund `__. The goal was to analyze barriers to decarbonisation in **China**, **India**, **Colombia**, **Chile**, **Kazakhstan**, **Bangladesh**, **Vietnam** and **Thailand** for power system planning in deep decarbonisation scenarios. -* Investigation of the energy transition pathways for **Nigeria** published in a `research paper `__ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behavior and one representing a decarbonized 2060 scenario. +* Investigation of the energy transition pathways for **Nigeria** published in a `research paper `__ by a joint team of PyPSA-Earth developers. Capabilities of PyPSA-Earth model has been showcased by considering two least-cost power system optimizations, one for 2020 to reproduce the historical system behavior and one representing a decarbonised 2060 scenario. * Modeling of the power sector in **Saudi Arabia** published as `MSc dissertation `__ in Edinburgh University. This has been a first attempt to predict the energy system for Saudi Arabia in 2060, with net zero emissions. The models in this study include a base model for validation, a model for 2030, to compare to the country’s Vision 2030 and as a transition phase, and two models for 2060 net zero, one is fully renewable and the other utilizing carbon capture and storage technology. The outputs of the work are available in the `repository `__. From e37a0535da347c8b2bc1747dcaf32249a1a76110 Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 23 Dec 2024 15:41:05 +0100 Subject: [PATCH 086/113] Update a list of contirbutors --- doc/project_structure_and_credits.rst | 31 ++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/doc/project_structure_and_credits.rst b/doc/project_structure_and_credits.rst index 268fcde23..fcdf6fdbd 100644 --- a/doc/project_structure_and_credits.rst +++ b/doc/project_structure_and_credits.rst @@ -23,13 +23,12 @@ The structure might be adjusted in future: - Director (`Max Parzen `_) - `Co-Director `_ (`Davide Fioriti `_) -- `PyPSA-Earth leader `_ (temporary Davide and Max) -- PyPSA-Earth-Sec leader (`Hazem Abdel-Khalek `_ and `Leon Schumm `_) +- `PyPSA-Earth leaders `_ (Davide, Katia, Max) +- PyPSA-Earth-Sec leaders (`Hazem Abdel-Khalek `_ and `Leon Schumm `_, Eddy Jalbout, Leon Schumm) - `AI and demand leader `_ (`Lukas Franken `_) - `Outreach leader `_ (`Stuart James `_) - `Finance leader `_ (Currently not assigned) -- Western Asia Coordinator (Emre Yorat and `Kasım Zor `_) -- Central Asia (`Ekatarina Fedotova `_) +- Western Asia Coordinators (Emre Yorat and `Kasım Zor `_) .. _credits: @@ -45,22 +44,30 @@ listed on the `PyPSA meets Earth team page Date: Mon, 23 Dec 2024 17:01:35 +0100 Subject: [PATCH 087/113] Update a list of contirbutors and the project structure --- doc/project_structure_and_credits.rst | 61 +++++++++++++++++++-------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/doc/project_structure_and_credits.rst b/doc/project_structure_and_credits.rst index fcdf6fdbd..94d6338c5 100644 --- a/doc/project_structure_and_credits.rst +++ b/doc/project_structure_and_credits.rst @@ -10,56 +10,80 @@ Project structure and credits The PyPSA-Earth model is maintained by the members of the PyPSA meets Earth initiative. We implemented in the initiative a project structure to enhance collaboration and give people responsibilities. -Therefore, the structure is not here to dictate, it is rather a structure that let people organise and think from different angles about the project. +Therefore, the structure is not here to dictate, it is rather a structure that let people organize and think from different angles about the project. We all have the same goal. We want to create a long-term maintained, supported and steadily improving energy system model that is useful for industry and research. Every person helping towards achieving this goal is listed in the credits. .. _project_structure: Project structure -==================== +================== -The structure might be adjusted in future: +PyPSA-meets-Earth initiative is build upon the code base which can be accessed from the initiative repository on GitHub. A list below aims to give a quick overview: + +- `PyPSA Earth `__ is a model for power or sector-coupled system modeling for any country of the world +- `PyPSA Distribution `__ is an optimization model for micro-grids and distribution grids with an automated extraction of the inputs needed to build grid topology and electricity load profiles https://github.com/pypsa-meets-earth/pypsa-distribution +- `Earth OSM `__ provides a toolset to download, pre-process and export energy-relevant infrastructure data from OpenStreetMap (OSM) + +The list is by no means exhaustive and is growing continuously. Feel free to explore more looking into the repositories available in `PyPSA meets Earth organization `__ and don't forget to give us a star ;). The contributions are very welcome to enhance the description of the available projects! + + +Track leaders +============== - Director (`Max Parzen `_) - `Co-Director `_ (`Davide Fioriti `_) -- `PyPSA-Earth leaders `_ (Davide, Katia, Max) -- PyPSA-Earth-Sec leaders (`Hazem Abdel-Khalek `_ and `Leon Schumm `_, Eddy Jalbout, Leon Schumm) -- `AI and demand leader `_ (`Lukas Franken `_) +- `PyPSA-Earth leaders `_ (Davide, Katia, Hazem, Max) +- Sector-coupling leaders (`Hazem Abdel-Khalek `_ and `Leon Schumm `_, Eddy Jalbout) +- Climate modeling leader (Katia) +- Coordinators of Regional Studies - `Outreach leader `_ (`Stuart James `_) -- `Finance leader `_ (Currently not assigned) - Western Asia Coordinators (Emre Yorat and `Kasım Zor `_) +- Africa Coordinator (Emmanuel Bolarinwa) +- South America Coordinator (Carlos Fernandez) +- `AI and demand leader `_ (`Lukas Franken `_) .. _credits: Credits ============= -The list below is outdated (December 2022). For maintenance reasons, -we refer the reader to the project website and GitHub repositories -listed on the `PyPSA meets Earth team page `_. +PyPSA-meets-Earth is an independent research and software development initiative which is developing thanks to the efforts of code and non-code contributors and users of our models. Every discussion comment, bug report and typo fixed count and are precious contributions into the community knowledge base. -(sorted by continent, country, institution) +The lists below is an attempt to give the credits to the contributors, though it's maintained manually, and may be outdated and incomplete. Feel free to reach out if you see the need for changes! A more complete version of the contributors list is available in our code repositories, e.g. `pypsa earth `__, `pypsa kz data `__, `pypsa distribution `__ or `earth osm `__. Points of contacts are listed on the site `PyPSA meets Earth team page `_. -Code Team ----------- -- Davide Fioriti (University of Pisa, Italy) +Contributors Team +----------------- -- Hazem Abdel-Khalek (Frauenhofer Institution for Energy IEG, Germany) +- Davide Fioriti (University of Pisa, Italy) +- Yerbol Akhmetov - Eddy Jalbout +- Hazem Abdel-Khalek (Frauenhofer Institution for Energy IEG, Germany) - Max Parzen (University of Edinburgh, United Kingdom) - Emre Yorat - Matin Mahmood (University of Edinburgh, United Kingdom) +- Daniele Lerede +- Alexander Meisinger (Ostbayerische Technische Hochschule Regensburg, Germany) - Emmanuel Bolarinwa -- Leon Schumm -- Fabian Neumann (TU Berlin, Germany) +- Lukas Trippe (Technical University of Berlin, Germany) +- Leon Schumm (Ostbayerische Technische Hochschule Regensburg, Germany) +- Sermisha Narayana +- Fabian Neumann (Technical University of Berlin, Germany) +- Carlos Fernandez - Anton Achhammer +- Arizeo Salac - Johannes Hampp (Justus Liebig University Giessen, Germany) -- Yerbol Akhmetov - Ekaterina Fedotova +- Muhammad Ilyas +- Lukas Franken (University of Edinburgh, United Kingdom) +- Jess Ryan - Stephen Lee (Massachusetts Institute of Technology, United States) +- Albert Solà Vilalta +- Martha Frysztacki - Denise Giubilato +- Anas Algarei +- Siddharth Krishna - Dahunsi Okekunle (Energy Market and Regulatory Consultants aka. EMRC, Nigeria) - Nse-Abasi Ayara (University of Abuja, Nigeria) - Jarrad Wright (Council for Scientific Research and Industry CSIR, South Africa) @@ -74,6 +98,7 @@ Outreach Team - Stuart James (VDMA, Germany) - Max Parzen (University of Edinburgh, United Kingdom) +- Tosin George - Mousa Zerai (University of Edinburgh, United Kingdom) - Ilaria Capelli (EY, Italy) - Rebecca Grant (University of Edinburgh, United Kingdom) From 62a4bfaddcab2010e2c187bcfb905ada8b869f3b Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 17:50:44 +0100 Subject: [PATCH 088/113] update FT to account for Hydrogen and Electricity input (#1226) * update FT to account for Hydrogen and electricity input * revert implementation * include electricity bus * added electricity bus * revert snakemock * Update release_notes.rst * revert efficiency for FT --- doc/release_notes.rst | 1 + scripts/prepare_sector_network.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0156ec70e..0f29269f9 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -18,6 +18,7 @@ This part of documentation collects descriptive release notes to capture the mai **Minor Changes and bug-fixing** +* Added electricity bus to Fischer-Tropsch in prepare_sector_network.py `PR #1226 `__ PyPSA-Earth 0.5.0 diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 796125b95..880dafef7 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -170,6 +170,7 @@ def H2_liquid_fossil_conversions(n, costs): bus0=spatial.nodes + " H2", bus1=spatial.oil.nodes, bus2=spatial.co2.nodes, + bus3=spatial.nodes, carrier="Fischer-Tropsch", efficiency=costs.at["Fischer-Tropsch", "efficiency"], capital_cost=costs.at["Fischer-Tropsch", "fixed"] @@ -178,6 +179,8 @@ def H2_liquid_fossil_conversions(n, costs): ], # Use efficiency to convert from EUR/MW_FT/a to EUR/MW_H2/a efficiency2=-costs.at["oil", "CO2 intensity"] * costs.at["Fischer-Tropsch", "efficiency"], + efficiency3=-costs.at["Fischer-Tropsch", "electricity-input"] + / costs.at["Fischer-Tropsch", "hydrogen-input"], p_nom_extendable=True, p_min_pu=options.get("min_part_load_fischer_tropsch", 0), lifetime=costs.at["Fischer-Tropsch", "lifetime"], From 236e0d0064bd3fdb2d3577274e13057da3233a9a Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 23 Dec 2024 18:15:17 +0100 Subject: [PATCH 089/113] Bypass external urls (#1257) * Update bundles with gadm-like to bypass gadm dependency * Minor fixes * Restore automatic emission download * Add release note * Add description to bundles --- configs/bundle_config.yaml | 43 ++++++++++++++++++++++++++++++++++---- doc/release_notes.rst | 2 ++ scripts/prepare_network.py | 5 +++-- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 7d1f55103..2f2824a54 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -36,11 +36,15 @@ databundles: category: data destination: "data" urls: - zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_NGBJ.zip?download=1 - gdrive: https://drive.google.com/file/d/1Vb1ISjhy7iwTTZYeezGd6S4nLt-EDGme/view?usp=drive_link + zenodo: https://sandbox.zenodo.org/records/145504/files/bundle_tutorial_NGBJ_with_gadmlike.zip?download=1 + gdrive: https://drive.google.com/file/d/12K03Epx3O9o-IQLh9afzCQyT-nMKWM3P/view?usp=drive_link output: - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + - data/gadm/gadm41_NGA/gadm41_NGA.gpkg # needed in build_shapes + - data/gadm/gadm41_BEN/gadm41_BEN.gpkg # needed in build_shapes + - data/gadm/gadm36_NGA/gadm36_NGA.gpkg # needed in sector-coupled model + - data/gadm/gadm36_BEN/gadm36_BEN.gpkg # needed in sector-coupled model # tutorial bundle specific for Botswana only bundle_tutorial_BW: @@ -49,11 +53,13 @@ databundles: category: data destination: "data" urls: - zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_BW.zip?download=1 - gdrive: https://drive.google.com/file/d/19IXvTD8gVSzgTInL85ta7QjaNI8ZPCCY/view?usp=drive_link + zenodo: https://sandbox.zenodo.org/records/145504/files/bundle_tutorial_BW_with_gadmlike.zip?download=1 + gdrive: https://drive.google.com/file/d/1YbbYGs1NsSsZYqNX1g1Jo-iJzt5m-81c/view?usp=drive_link output: - data/gebco/GEBCO_2021_TID.nc - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + - data/gadm/gadm41_BWA/gadm41_BWA.gpkg # needed in build_shapes + - data/gadm/gadm36_BWA/gadm36_BWA.gpkg # needed in sector-coupled model # tutorial bundle specific for Morocco only bundle_tutorial_MA: @@ -364,3 +370,32 @@ databundles: urls: protectedplanet: https://d1gam3xoknrgr2.cloudfront.net/current/WDPA_{month:s}{year:d}_Public_shp.zip output: [data/landcover/world_protected_areas/*] + + # Backup tutorial bundles with no gadm-like data; for reference: + # https://github.com/pypsa-meets-earth/pypsa-earth/issues/1258 + # + # # tutorial bundle specific for Nigeria and Benin only, without gadm-like data + # bundle_tutorial_NGBJ: + # countries: [NG, BJ] + # tutorial: true + # category: data + # destination: "data" + # urls: + # zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_NGBJ.zip?download=1 + # gdrive: https://drive.google.com/file/d/1Vb1ISjhy7iwTTZYeezGd6S4nLt-EDGme/view?usp=drive_link + # output: + # - data/gebco/GEBCO_2021_TID.nc + # - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif + + # # tutorial bundle specific for Botswana only, without gadm-like data + # bundle_tutorial_BW: + # countries: [BW] + # tutorial: true + # category: data + # destination: "data" + # urls: + # zenodo: https://sandbox.zenodo.org/records/3853/files/bundle_tutorial_BW.zip?download=1 + # gdrive: https://drive.google.com/file/d/19IXvTD8gVSzgTInL85ta7QjaNI8ZPCCY/view?usp=drive_link + # output: + # - data/gebco/GEBCO_2021_TID.nc + # - data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0f29269f9..bd61a857e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -20,6 +20,8 @@ This part of documentation collects descriptive release notes to capture the mai * Added electricity bus to Fischer-Tropsch in prepare_sector_network.py `PR #1226 `__ +* Update BW, NG and BJ tutorial databundles to include gadm-like sources from geoboundaries `PR #1257 `__ + PyPSA-Earth 0.5.0 ================= diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 3d6c73cb8..dc03380db 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -325,6 +325,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): clusters="4", ll="c1", opts="Co2L-4H", + configfile="test/config.sector.yaml", ) configure_logging(snakemake) @@ -372,10 +373,10 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): co2limit = co2limit * float(m[0]) logger.info("Setting CO2 limit according to emission base year.") elif len(m) > 0: - co2limit = float(m[0]) * snakemake.params.electricity["co2base"] + co2limit = float(m[0]) * float(snakemake.params.electricity["co2base"]) logger.info("Setting CO2 limit according to wildcard value.") else: - co2limit = snakemake.params.electricity["co2limit"] + co2limit = float(snakemake.params.electricity["co2limit"]) logger.info("Setting CO2 limit according to config value.") add_co2limit(n, co2limit, Nyears) break From b8a99aba98aada0deb9780e0f0e17ad1148976f2 Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Mon, 23 Dec 2024 18:19:41 +0100 Subject: [PATCH 090/113] Update release_note --- doc/release_notes.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 23a18ddd6..fac4f8272 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -16,6 +16,14 @@ This part of documentation collects descriptive release notes to capture the mai * Include option in the config to allow for custom airport data `PR #1241 `__ +* Computational improvements of build_osm_network `PR #845 `__ + +* Boost computational performances of set_lines_ids with cKDTree by scipy `PR #806 `__ + +* Boost computational performances of set_substation_ids using DBSCAN `PR #799 `__ + +* Boost computational performances of fix_overpassing_line `PR #807 `__ + **Minor Changes and bug-fixing** * Added electricity bus to Fischer-Tropsch in prepare_sector_network.py `PR #1226 `__ @@ -122,8 +130,6 @@ PyPSA-Earth 0.4.0 * Add an option to use csv format for custom demand imports. `PR #995 `__ -* Computational improvements of build_osm_network `PR #845 `__ - **Minor Changes and bug-fixing** * Minor bug-fixing to run the cluster wildcard min `PR #1019 `__ @@ -170,12 +176,6 @@ PyPSA-Earth 0.3.0 * Use `new CC0 v1 dataset `__ for the natura input and automate download of WDPA protected planet data `PR #913 `__ -* Boost computational performances of set_lines_ids with cKDTree by scipy `PR #806 `__ - -* Boost computational performances of set_substation_ids using DBSCAN `PR #799 `__ - -* Boost computational performances of fix_overpassing_line `PR #807 `__ - **Minor Changes and bug-fixing** * Revise databundles and improve logging in retrieve_databundle `PR #928 `__ From 3dc6ddcd0421581ca31c6ef0016530e187dece41 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 23 Dec 2024 18:35:56 +0100 Subject: [PATCH 091/113] update gdrive link (#1259) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15556eab2..34b998185 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ by [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pypsa-meets-earth/pypsa-earth/main.svg)](https://results.pre-commit.ci/latest/github/pypsa-meets-earth/pypsa-earth/main) [![Discord](https://img.shields.io/discord/911692131440148490?logo=discord)](https://discord.gg/AnuJBk23FU) -[![Google Drive](https://img.shields.io/badge/Google%20Drive-4285F4?style=flat&logo=googledrive&logoColor=white)](https://drive.google.com/drive/folders/1U7fgktbxlaGzWxT2C0-Xv-_ffWCxAKZz) +[![Google Drive](https://img.shields.io/badge/Google%20Drive-4285F4?style=flat&logo=googledrive&logoColor=white)](https://drive.google.com/drive/folders/13Z8Y9zgsh5IZaDNkkRyo1wkoMgbdUxT5?usp=sharing) **PyPSA-Earth: A Global Sector-Coupled Open-Source Multi-Energy System Model** From a0fa30f2c1179797b3ad50913f8380cb7ec008f2 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 23 Dec 2024 19:23:48 +0100 Subject: [PATCH 092/113] Fix docs (#1260) * Update env requirements for doc * Update documentation environment --- doc/requirements.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index d2b518fd8..1dd52eaaf 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -8,7 +8,7 @@ sphinx_book_theme sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 -pypsa +pypsa >=0.24, <0.25 vresutils>=0.3.1 powerplantmatching>=0.5.5 atlite>=0.2.9 @@ -33,7 +33,9 @@ gitpython chaospy numba ruamel.yaml<=0.17.26 -earth-osm>=0.1.0, <0.2.0 +earth-osm>=2.3.post1 reverse-geocode pyDOE2 -# graphviz +graphviz + +fake_useragent From afd97122d4db7946406aa771455903d1bf585fb0 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:24:41 +0100 Subject: [PATCH 093/113] Include dev container and docker feature --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index bd61a857e..03b08a61e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -15,6 +15,7 @@ This part of documentation collects descriptive release notes to capture the mai * Include option in the config to allow for custom airport data `PR #1241 `__ +* Added Dev Containers and docker as an option to get started with pypsa-earth `PR #1228 `__ **Minor Changes and bug-fixing** From 5c5c2b0138cc7d526736b571b48b174eac78b1c1 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Mon, 23 Dec 2024 20:29:07 +0100 Subject: [PATCH 094/113] Restore environment (#1048) * Remove a constraint on geopandas * Restore earth-osm conda --------- Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Co-authored-by: Davide Fioriti --- envs/environment.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index b82cf3aa2..96278f6f2 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -17,9 +17,7 @@ dependencies: # currently the packages are being installed with pip # need to move back to conda once the issues will be resolved - powerplantmatching>=0.5.19 -# - earth-osm>=2.1 -# until the release will incorporate all the fixes needed -# to work with CDS beta +- earth-osm>=2.3.post1 - atlite>=0.3 # Dependencies of the workflow itself @@ -35,8 +33,8 @@ dependencies: # starting from 1.3.5 numpoly requires numpy>2.0 which leads to issues - numpoly<=1.3.4 - pandas -- geopandas>=0.11.0, <=0.14.3 -- fiona<1.10.0 +- geopandas>=0.11.0 +- fiona!=1.8.22 - xarray>=2023.11.0, <2023.12.0 - netcdf4 - networkx @@ -86,7 +84,6 @@ dependencies: - gurobi - pip: - - earth-osm==2.2 # until conda release it out for earth-osm - git+https://github.com/davide-f/google-drive-downloader@master # google drive with fix for virus scan - tsam>=1.1.0 - chaospy # lastest version only available on pip From 4760a3966629efccd33284c18fabbcbfbdb758d1 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:36:43 +0100 Subject: [PATCH 095/113] Update welcome message --- .devcontainer/welcome-message.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt index afcec3035..709322166 100644 --- a/.devcontainer/welcome-message.txt +++ b/.devcontainer/welcome-message.txt @@ -6,11 +6,11 @@ 👋 Welcome to the PyPSA-Earth Development Environment! -We’re excited to have you here! This setup allows you to contribute to PyPSA-Earth effortlessly using a development container in VS Code. +We’re excited to have you here! This setup allows you to use and contribute to PyPSA-Earth using a development container in VS Code. 📖 Getting Started for New Users - • For a step-by-step guide on setting up your environment, debugging, and making your first contribution, refer to the PyPSA-Earth README. It covers everything you need to know as a newcomer. + • For a step-by-step guide on setting up your environment, debugging, and making your first contribution, refer to the PyPSA-Earth README here: (https://github.com/pypsa-meets-earth/pypsa-earth/blob/main/README.md). It covers everything you need to know as a newcomer. • The configuration files for the development container are located in the .github/.devcontainer folder. 💡 Tips for New Users @@ -20,4 +20,4 @@ We’re excited to have you here! This setup allows you to contribute to PyPSA-E 🚀 Start Exploring and Happy Coding! -Don’t hesitate to reach out if you need help—our community is here to support you. +Don’t hesitate to reach out if you need help—our community is here to support you. You can access our discord server here: https://discord.gg/AnuJBk23FU From c931b513e00c7f013da0908a12201277b2d0e3e6 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:41:06 +0100 Subject: [PATCH 096/113] Update devcontainer.yml --- .github/workflows/devcontainer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 12288a6b1..85de566fe 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -30,4 +30,3 @@ jobs: cacheFrom: ghcr.io/${{ github.repository }}-dev-env imageName: ghcr.io/${{ github.repository }}-dev-env push: always - # noCache: true From 1c3e3f4a6ef598e12dcbf582628fcea0477df2d0 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:45:18 +0100 Subject: [PATCH 097/113] Update Docker.MD --- Docker.MD | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Docker.MD b/Docker.MD index c2db2aaa7..75e99f0cc 100644 --- a/Docker.MD +++ b/Docker.MD @@ -17,7 +17,11 @@ Ensure Docker is installed on your system. Follow the instructions for your oper - **Linux**: [Docker for Linux](https://docs.docker.com/desktop/install/linux/) - **MacOS**: [Docker for MacOS](https://docs.docker.com/desktop/install/mac-install/) -### 2. Install GitHub Desktop +### 2. Install Visual Studio Code(VSC) + +You can use the link [here](https://code.visualstudio.com/download) to install Visual Studio Code on your operating system. Ensure to select the most compatible file for your operating system. + +### 3. Install GitHub Desktop You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install GitHub Desktop for your operating system from [here](https://desktop.github.com/download/). From 65c1ca3e3c85cb4c370769b4af0b451e77bf6414 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:48:55 +0100 Subject: [PATCH 098/113] Update Docker.MD --- Docker.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker.MD b/Docker.MD index 75e99f0cc..16e96e0dc 100644 --- a/Docker.MD +++ b/Docker.MD @@ -45,7 +45,7 @@ You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install G ### Step 2: Rebuild and Open in Container 1. Open the cloned repository in **VSCode**. -2. Click on the green icon in the bottom left corner of the VSCode window. +2. Click on the icon located at the bottom left corner of the VSCode window. 3. Select **Reopen in Container**. 4. Wait for the container to build and for the repository to open in the container. From 310635648f739b11d547fc2cb11a5d2dea0365a7 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:49:48 +0100 Subject: [PATCH 099/113] Update Docker.MD --- Docker.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker.MD b/Docker.MD index 16e96e0dc..24e8a9d84 100644 --- a/Docker.MD +++ b/Docker.MD @@ -34,7 +34,7 @@ You will also need GitHub Desktop to clone the PyPSA-Earth repository. Install G 3. Paste the following URL in the URL field: ```bash - https://github.com/drifter089/pypsa-earth.git + https://github.com/pypsa-meets-earth/pypsa-earth.git ``` 4. Click on **Clone**. From 5c96022de4f26bcc7e22d47c03373049daf50d4a Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:53:34 +0100 Subject: [PATCH 100/113] revert config.tutorial.yaml --- config.tutorial.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 9c2b87d1a..7ada63032 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -9,7 +9,6 @@ tutorial: true countries: ["NG", "BJ"] enable: - retrieve_databundle: true build_natura_raster: true progress_bar: false From 0c042a0a4268fe0ac463c24c25d17d8416a69a54 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Mon, 23 Dec 2024 20:57:24 +0100 Subject: [PATCH 101/113] Update docker_containers.rst --- doc/docker_containers.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/docker_containers.rst b/doc/docker_containers.rst index bc55423b7..68a944cb9 100644 --- a/doc/docker_containers.rst +++ b/doc/docker_containers.rst @@ -21,9 +21,11 @@ Steps: Ensure Docker is installed on your system. -2. Install GitHub Desktop for your OS `here `_. +2. You can use the link `here `_ to install Visual Studio Code on your operating system. Ensure to select the most compatible file for your operating system. -3. Clone the repository: +3. Install GitHub Desktop for your OS `here `_. + +4. Clone the repository: * Open GitHub Desktop. * Click on "File" in the top left corner. * Click on "Clone Repository". @@ -31,7 +33,7 @@ Steps: .. code:: bash - https://github.com/drifter089/pypsa-earth.git + https://github.com/pypsa-meets-earth/pypsa-earth.git * Click on "Clone". * Choose the location where you want to save the repository. @@ -40,9 +42,9 @@ Steps: The repository will be cloned to your local machine. -4. Rebuild and open in a container: +5. Rebuild and open in a container: * Open the repository in VSCode. - * Click on the green icon in the bottom left corner of the VSCode window. + * Click on the icon in the far bottom left corner of the VSCode window. * Click on "Reopen in Container". * Wait for the container to build and open the repository in the container. From 6675503001bd95dceb2bebae7a3b5c9fd22783e1 Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 23 Dec 2024 22:39:33 +0100 Subject: [PATCH 102/113] Add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0156ec70e..a4b36b0e3 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -15,6 +15,8 @@ This part of documentation collects descriptive release notes to capture the mai * Include option in the config to allow for custom airport data `PR #1241 `__ +* Add a list of PyPSA-Earth applications in academic and industrial projects `PR #1255 `__ + **Minor Changes and bug-fixing** From 453673912256c1d438dde64dca2edfa1383d9efa Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Tue, 24 Dec 2024 00:07:32 +0100 Subject: [PATCH 103/113] Fix doc errors and warnings --- config.default.yaml | 1 + doc/configtables/build_shape_options.csv | 20 ++++++++++---------- doc/index.rst | 2 ++ scripts/_helpers.py | 14 ++++++++------ 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 47053eb25..715a8a4a9 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -355,6 +355,7 @@ renewable: csp_model: advanced # simple or advanced # TODO: Needs to be adjusted for Africa. +# Costs Configuration costs: year: 2030 version: v0.6.2 diff --git a/doc/configtables/build_shape_options.csv b/doc/configtables/build_shape_options.csv index 16f30ff0c..0008aa78b 100644 --- a/doc/configtables/build_shape_options.csv +++ b/doc/configtables/build_shape_options.csv @@ -1,10 +1,10 @@ -,Unit,Values,Description,,,, -gadm_layer_id,,"""{0",1,"2}""","""GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country",1: region/county-like,"2: municipality-like.""" -simplify_gadm,bool," ""{True"," False}""",True: shape polygons are simplified else no,,, -update_file, bool," ""{True"," False}"""," ""True: all input files are downloaded again and replace the existing files.""",,, -out_logging, bool," ""{True"," False}"""," ""True: Logging is printed in the console.""",,, -year,," ""past year; e.g. YYYY"""," ""Reference year used to derive shapes"," info on population and info on GDP.""",,, -nprocesses, int,," ""Number of processes to be used in build_shapes.""",,,, -worldpop_method,," ""{""standard"""," ""api"""," false}""","""Specifies how population is added to every shape: ""standard"" pulls from web 1kmx1km raster; ""api"" pulls from API 100mx100m raster; false (not ""false"") no population addition to shape. This is useful when generating only cutout.""",, -gdp_method,," ""{""standard"""," false}""","""Specifies how GDP is added to every shape: ""standard"" pulls from web 1x1km raster; false (not ""false"") no gdp addition to shape. This is useful when generating only cutout.""",,, -contended_flag,," ""{""set_by_country"""," ""drop""}"""," ""Specifies what to do with contended countries: ""set_by_country"" assigns the contended areas to the countries according to the GADM database; ""drop"" drops the contended areas from the model.""",,, +,Unit,Values,Description +gadm_layer_id,,"{0,1,2}","GADM level area used for the gadm_shapes. Codes are country-dependent but roughly: 0: country,1: region/county-like,2: municipality-like." +simplify_gadm,bool,"{True, False}","True: shape polygons are simplified else no" +update_file, bool,"{True, False}","True: all input files are downloaded again and replace the existing files." +out_logging, bool,"{True, False}","True: Logging is printed in the console." +year,,"past year; e.g. YYYY","Reference year used to derive shapes info on population and info on GDP." +nprocesses, int,,"Number of processes to be used in build_shapes." +worldpop_method,,"{standard,api, false}","""Specifies how population is added to every shape: ""standard"" pulls from web 1kmx1km raster; ""api"" pulls from API 100mx100m raster; false (not ""false"") no population addition to shape. This is useful when generating only cutout.""" +gdp_method,,"""{""standard"",""false""}""","""Specifies how GDP is added to every shape: ""standard"" pulls from web 1x1km raster; false (not ""false"") no gdp addition to shape. This is useful when generating only cutout.""" +contended_flag,,"""{""set_by_country"",""drop""}""","""Specifies what to do with contended countries: ""set_by_country"" assigns the contended areas to the countries according to the GADM database; ""drop"" drops the contended areas from the model.""" diff --git a/doc/index.rst b/doc/index.rst index 3385411be..f9328bd09 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -182,6 +182,7 @@ Documentation * :doc:`release_notes` * :doc:`how_to_contribute` +* :doc:`how_to_docs` * :doc:`software_hints` * :doc:`learning_materials` @@ -192,6 +193,7 @@ Documentation release_notes how_to_contribute + how_to_docs software_hints learning_materials diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 36cf0d95c..ddb6fda5f 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1132,18 +1132,20 @@ def get_country(target, **keys): target: str Desired type of country code. Examples: - - 'alpha_3' for 3-digit - - 'alpha_2' for 2-digit - - 'name' for full country name + - 'alpha_3' for 3-digit + - 'alpha_2' for 2-digit + - 'name' for full country name keys: dict Specification of the country name and reference system. Examples: - - alpha_3="ZAF" for 3-digit - - alpha_2="ZA" for 2-digit - - name="South Africa" for full country name + - alpha_3="ZAF" for 3-digit + - alpha_2="ZA" for 2-digit + - name="South Africa" for full country name + Returns ------- country code as requested in keys or np.nan, when country code is not recognized + Example of usage ------- - Convert 2-digit code to 3-digit codes: get_country('alpha_3', alpha_2="ZA") From d566e988a93a881dc24fb53aa5312f2ef8ce7db3 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Dec 2024 10:03:58 +0100 Subject: [PATCH 104/113] Put the path into snakemake inputs --- Snakefile | 3 ++- scripts/build_base_industry_totals.py | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Snakefile b/Snakefile index 41d8952fc..a0e1a3a75 100644 --- a/Snakefile +++ b/Snakefile @@ -1875,8 +1875,9 @@ rule build_base_industry_totals: #default data countries=config["countries"], other_industries=config["demand_data"]["other_industries"], input: + #os.path.dirname(snakemake.input["transactions_path"]) + "/demand/unsd/data/" #industrial_production_per_country="data/industrial_production_per_country.csv", - #unsd_path="data/demand/unsd/data/", + unsd_path="data/demand/unsd/data/", energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv", transactions_path="data/unsd_transactions.csv", output: diff --git a/scripts/build_base_industry_totals.py b/scripts/build_base_industry_totals.py index 4c33aaaad..1e5eda9a8 100644 --- a/scripts/build_base_industry_totals.py +++ b/scripts/build_base_industry_totals.py @@ -117,10 +117,7 @@ def create_industry_base_totals(df): renaming_dit = transaction.set_index("Transaction")["clean_name"].to_dict() clean_industry_list = list(transaction.clean_name.unique()) - # TODO Check with the sector-coupled modelling experts - unsd_path = ( - os.path.dirname(snakemake.input["transactions_path"]) + "/demand/unsd/data/" - ) + unsd_path = snakemake.input.unsd_path # Get the files from the path provided in the OP all_files = list(Path(unsd_path).glob("*.txt")) From e7dd6509db348a0a5e58fc0dd4fdf77c77446aa2 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Tue, 24 Dec 2024 15:00:26 +0100 Subject: [PATCH 105/113] Setup environment for release (#1263) * Restore env gpd and fiona limitations * Update pinned environment files for all platforms * Cleaning pinned environment --- envs/environment.yaml | 4 +- envs/linux-pinned.yaml | 109 +++++++++++++++++++-------------------- envs/macos-pinned.yaml | 96 +++++++++++++++++----------------- envs/windows-pinned.yaml | 97 +++++++++++++++++----------------- 4 files changed, 153 insertions(+), 153 deletions(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 96278f6f2..4c172b36e 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -33,8 +33,8 @@ dependencies: # starting from 1.3.5 numpoly requires numpy>2.0 which leads to issues - numpoly<=1.3.4 - pandas -- geopandas>=0.11.0 -- fiona!=1.8.22 +- geopandas>=0.11.0, <=0.14.3 +- fiona!=1.8.22, <1.10.0 - xarray>=2023.11.0, <2023.12.0 - netcdf4 - networkx diff --git a/envs/linux-pinned.yaml b/envs/linux-pinned.yaml index fadd8a2fe..53b2b8f48 100644 --- a/envs/linux-pinned.yaml +++ b/envs/linux-pinned.yaml @@ -27,16 +27,16 @@ dependencies: - atk-1.0=2.38.0 - atlite=0.3.0 - attr=2.5.1 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -50,30 +50,30 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 - cftime=1.6.4 - charset-normalizer=3.4.0 -- click=8.1.7 +- click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 - cloudpickle=3.1.0 @@ -94,23 +94,24 @@ dependencies: - cpp-expected=1.1.0 - cycler=0.12.1 - cyrus-sasl=2.1.27 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 - dbus=1.13.6 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 +- earth-osm=2.3.post1 - entrypoints=0.4 - entsoe-py=0.6.11 - et_xmlfile=2.0.0 @@ -120,7 +121,7 @@ dependencies: - filelock=3.16.1 - fiona=1.9.6 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -133,7 +134,7 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - gdk-pixbuf=2.42.12 @@ -144,8 +145,8 @@ dependencies: - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - gettext=0.22.5 - gettext-tools=0.22.5 - gflags=2.2.2 @@ -166,7 +167,7 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=9.0.0 +- harfbuzz=10.1.0 - hdf4=4.2.15 - hdf5=1.14.3 - holoviews=1.20.0 @@ -174,7 +175,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -182,14 +183,14 @@ dependencies: - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 - importlib_resources=6.4.5 -- inflate64=1.0.0 +- inflate64=1.0.1 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 -- jinja2=3.1.4 +- jinja2=3.1.5 - joblib=1.4.2 - json-c=0.18 - json5=0.10.0 @@ -200,10 +201,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -229,12 +230,12 @@ dependencies: - libbrotlienc=1.1.0 - libcap=2.71 - libcblas=3.9.0 -- libclang-cpp19.1=19.1.5 -- libclang13=19.1.5 +- libclang-cpp19.1=19.1.6 +- libclang13=19.1.6 - libcrc32c=1.1.2 - libcups=2.3.3 - libcurl=8.11.1 -- libdeflate=1.22 +- libdeflate=1.23 - libdrm=2.4.124 - libedit=3.1.20191231 - libegl=1.7.0 @@ -282,14 +283,14 @@ dependencies: - liblapack=3.9.0 - liblapacke=3.9.0 - libllvm14=14.0.6 -- libllvm19=19.1.5 +- libllvm19=19.1.6 - liblzma=5.6.3 - liblzma-devel=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libnghttp2=1.64.0 - libnsl=2.0.1 -- libntlm=1.4 +- libntlm=1.8 - libogg=1.3.5 - libopenblas=0.3.28 - libopus=1.3.1 @@ -305,7 +306,7 @@ dependencies: - libsndfile=1.2.2 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.0.0 +- libspatialindex=2.1.0 - libspatialite=5.1.0 - libspral=2024.05.08 - libsqlite=3.47.2 @@ -318,7 +319,7 @@ dependencies: - libutf8proc=2.9.0 - libuuid=2.38.1 - libvorbis=1.3.7 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libxcb=1.17.0 - libxcrypt=4.4.36 - libxkbcommon=1.7.0 @@ -334,7 +335,7 @@ dependencies: - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -359,7 +360,7 @@ dependencies: - munkres=1.1.4 - mysql-common=9.0.1 - mysql-libs=9.0.1 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - ncurses=6.5 @@ -385,9 +386,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -412,6 +413,7 @@ dependencies: - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 +- protobuf=5.28.2 - psutil=6.1.0 - pthread-stubs=0.4 - ptyprocess=0.7.0 @@ -422,7 +424,7 @@ dependencies: - py7zr=0.22.0 - pyarrow=18.1.0 - pyarrow-core=18.1.0 -- pybcj=1.0.2 +- pybcj=1.0.3 - pycountry=24.6.1 - pycparser=2.22 - pycryptodomex=3.21.0 @@ -470,7 +472,7 @@ dependencies: - rtree=1.3.0 - ruamel.yaml=0.17.26 - ruamel.yaml.clib=0.2.8 -- s2n=1.5.9 +- s2n=1.5.10 - scikit-learn=1.6.0 - scipy=1.14.1 - seaborn=0.13.2 @@ -478,10 +480,10 @@ dependencies: - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - sip=6.7.12 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -500,7 +502,7 @@ dependencies: - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - toml=0.10.2 @@ -522,7 +524,7 @@ dependencies: - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 -- urllib3=2.2.3 +- urllib3=2.3.0 - validators=0.34.0 - virtualenv=20.28.0 - wcwidth=0.2.13 @@ -540,17 +542,16 @@ dependencies: - xerces-c=3.2.5 - xkeyboard-config=2.43 - xlrd=2.0.1 -- xorg-libice=1.1.1 -- xorg-libsm=1.2.4 +- xorg-libice=1.1.2 +- xorg-libsm=1.2.5 - xorg-libx11=1.8.10 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdamage=1.1.6 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - xorg-libxfixes=6.0.1 -- xorg-libxrender=0.9.11 -- xorg-libxxf86vm=1.1.5 -- xorg-xf86vidmodeproto=2.3.1 +- xorg-libxrender=0.9.12 +- xorg-libxxf86vm=1.1.6 - xyzservices=2024.9.0 - xz=5.6.3 - xz-gpl-tools=5.6.3 @@ -568,10 +569,8 @@ dependencies: - zstd=1.5.6 - pip: - chaospy==4.3.17 - - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 - - protobuf==5.29.1 + - highspy==1.9.0 - tsam==2.3.6 prefix: /usr/share/miniconda/envs/pypsa-earth diff --git a/envs/macos-pinned.yaml b/envs/macos-pinned.yaml index 96e4a8c09..38c73a2fc 100644 --- a/envs/macos-pinned.yaml +++ b/envs/macos-pinned.yaml @@ -24,16 +24,16 @@ dependencies: - async-lru=2.0.4 - atk-1.0=2.38.0 - atlite=0.3.0 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -47,30 +47,30 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 - cftime=1.6.4 - charset-normalizer=3.4.0 -- click=8.1.7 +- click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 - cloudpickle=3.1.0 @@ -91,22 +91,23 @@ dependencies: - cpp-expected=1.1.0 - cycler=0.12.1 - cyrus-sasl=2.1.27 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 +- earth-osm=2.3.post1 - entrypoints=0.4 - entsoe-py=0.6.11 - et_xmlfile=2.0.0 @@ -115,7 +116,7 @@ dependencies: - filelock=3.16.1 - fiona=1.9.6 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -128,7 +129,7 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - gdk-pixbuf=2.42.12 @@ -139,8 +140,8 @@ dependencies: - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - gflags=2.2.2 - giflib=5.2.2 - gitdb=4.0.11 @@ -155,7 +156,7 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=9.0.0 +- harfbuzz=10.1.0 - hdf4=4.2.15 - hdf5=1.14.3 - holoviews=1.20.0 @@ -163,7 +164,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -171,14 +172,14 @@ dependencies: - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 - importlib_resources=6.4.5 -- inflate64=1.0.0 +- inflate64=1.0.1 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 -- jinja2=3.1.4 +- jinja2=3.1.5 - joblib=1.4.2 - json-c=0.18 - json5=0.10.0 @@ -189,10 +190,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -214,8 +215,8 @@ dependencies: - libcblas=3.9.0 - libcrc32c=1.1.2 - libcurl=8.11.1 -- libcxx=19.1.5 -- libdeflate=1.22 +- libcxx=19.1.6 +- libdeflate=1.23 - libedit=3.1.20191231 - libev=4.33 - libevent=2.1.12 @@ -250,10 +251,10 @@ dependencies: - liblapacke=3.9.0 - libllvm14=14.0.6 - liblzma=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libnghttp2=1.64.0 -- libntlm=1.4 +- libntlm=1.8 - libopenblas=0.3.28 - libparquet=18.1.0 - libpng=1.6.44 @@ -265,14 +266,14 @@ dependencies: - libscotch=7.0.5 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.0.0 +- libspatialindex=2.1.0 - libspatialite=5.1.0 - libsqlite=3.47.2 - libssh2=1.11.1 - libthrift=0.21.0 - libtiff=4.7.0 - libutf8proc=2.9.0 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libxcb=1.17.0 - libxml2=2.13.5 - libxslt=1.1.39 @@ -280,14 +281,14 @@ dependencies: - libzlib=1.3.1 - linkify-it-py=2.0.3 - linopy=0.3.11 -- llvm-openmp=19.1.5 +- llvm-openmp=19.1.6 - llvmlite=0.43.0 - locket=1.0.0 - lxml=5.3.0 - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -309,7 +310,7 @@ dependencies: - mumps-include=5.7.3 - mumps-seq=5.7.3 - munkres=1.1.4 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - ncurses=6.5 @@ -334,9 +335,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -361,6 +362,7 @@ dependencies: - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 +- protobuf=5.28.2 - psutil=6.1.0 - pthread-stubs=0.4 - ptyprocess=0.7.0 @@ -370,7 +372,7 @@ dependencies: - py7zr=0.22.0 - pyarrow=18.1.0 - pyarrow-core=18.1.0 -- pybcj=1.0.2 +- pybcj=1.0.3 - pycountry=24.6.1 - pycparser=2.22 - pycryptodomex=3.21.0 @@ -424,9 +426,9 @@ dependencies: - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -445,7 +447,7 @@ dependencies: - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - tomli=2.2.1 @@ -466,7 +468,7 @@ dependencies: - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 -- urllib3=2.2.3 +- urllib3=2.3.0 - validators=0.34.0 - virtualenv=20.28.0 - wcwidth=0.2.13 @@ -478,7 +480,7 @@ dependencies: - xarray=2023.11.0 - xerces-c=3.2.5 - xlrd=2.0.1 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdmcp=1.1.5 - xyzservices=2024.9.0 - yaml=0.2.5 @@ -494,10 +496,8 @@ dependencies: - zstd=1.5.6 - pip: - chaospy==4.3.17 - - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 - - protobuf==5.29.1 + - highspy==1.9.0 - tsam==2.3.6 prefix: /Users/runner/miniconda3/envs/pypsa-earth diff --git a/envs/windows-pinned.yaml b/envs/windows-pinned.yaml index ca45bea29..4ac607945 100644 --- a/envs/windows-pinned.yaml +++ b/envs/windows-pinned.yaml @@ -14,6 +14,7 @@ channels: dependencies: - _openmp_mutex=4.5 - affine=2.4.0 +- ampl-asl=1.0.0 - amply=0.1.6 - anyio=4.7.0 - appdirs=1.4.4 @@ -23,16 +24,16 @@ dependencies: - asttokens=3.0.0 - async-lru=2.0.4 - atlite=0.3.0 -- attrs=24.2.0 +- attrs=24.3.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.1 -- aws-c-common=0.10.5 +- aws-c-common=0.10.6 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - aws-c-http=0.9.2 - aws-c-io=0.15.3 - aws-c-mqtt=0.11.0 -- aws-c-s3=0.7.5 +- aws-c-s3=0.7.7 - aws-c-sdkutils=0.2.1 - aws-checksums=0.2.2 - aws-crt-cpp=0.29.7 @@ -45,30 +46,30 @@ dependencies: - beautifulsoup4=4.12.3 - bleach=6.2.0 - blosc=1.21.6 -- bokeh=3.5.2 +- bokeh=3.6.2 - bottleneck=1.4.2 -- branca=0.7.2 +- branca=0.8.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - brotlicffi=1.1.0.0 - bzip2=1.0.8 -- c-ares=1.34.3 +- c-ares=1.34.4 - c-blosc2=2.15.2 -- ca-certificates=2024.8.30 +- ca-certificates=2024.12.14 - cached-property=1.5.2 - cached_property=1.5.2 - cairo=1.18.2 - capnproto=1.0.2 - cartopy=0.23.0 - cdsapi=0.7.5 -- certifi=2024.8.30 +- certifi=2024.12.14 - cffi=1.17.1 - cfgv=3.3.1 - cfitsio=4.4.1 - cftime=1.6.4 - charset-normalizer=3.4.0 -- click=8.1.7 +- click=8.1.8 - click-plugins=1.1.1 - cligj=0.7.2 - cloudpickle=3.1.0 @@ -83,22 +84,23 @@ dependencies: - cpp-expected=1.1.0 - cpython=3.10.16 - cycler=0.12.1 -- cytoolz=1.0.0 -- dask=2024.12.0 -- dask-core=2024.12.0 -- dask-expr=1.1.20 +- cytoolz=1.0.1 +- dask=2024.12.1 +- dask-core=2024.12.1 +- dask-expr=1.1.21 - datapi=0.1.1 - datashader=0.16.3 - datrie=0.8.2 -- debugpy=1.8.10 +- debugpy=1.8.11 - decorator=5.1.1 - defusedxml=0.7.1 - deprecation=2.1.0 - descartes=1.1.0 - distlib=0.3.9 -- distributed=2024.12.0 +- distributed=2024.12.1 - docutils=0.21.2 - dpath=2.2.0 +- earth-osm=2.3.post1 - entrypoints=0.4 - entsoe-py=0.6.11 - et_xmlfile=2.0.0 @@ -107,7 +109,7 @@ dependencies: - filelock=3.16.1 - fiona=1.9.6 - fmt=11.0.2 -- folium=0.19.0 +- folium=0.19.2 - font-ttf-dejavu-sans-mono=2.37 - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 @@ -120,7 +122,7 @@ dependencies: - freetype=2.12.1 - freexl=2.0.0 - fribidi=1.0.10 -- fsspec=2024.10.0 +- fsspec=2024.12.0 - future=1.0.0 - gdal=3.9.3 - geographiclib=2.0 @@ -130,8 +132,8 @@ dependencies: - geopy=2.4.1 - geos=3.13.0 - geotiff=1.7.3 -- geoviews=1.13.1 -- geoviews-core=1.13.1 +- geoviews=1.14.0 +- geoviews-core=1.14.0 - getopt-win32=0.1 - gitdb=4.0.11 - gitpython=3.1.43 @@ -146,7 +148,7 @@ dependencies: - gurobi=12.0.0 - h11=0.14.0 - h2=4.1.0 -- harfbuzz=9.0.0 +- harfbuzz=10.1.0 - hdf4=4.2.15 - hdf5=1.14.3 - holoviews=1.20.0 @@ -154,7 +156,7 @@ dependencies: - httpcore=1.0.7 - httpx=0.28.1 - humanfriendly=10.0 -- hvplot=0.11.1 +- hvplot=0.11.2 - hyperframe=6.0.1 - icu=75.1 - identify=2.6.3 @@ -162,14 +164,14 @@ dependencies: - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 - importlib_resources=6.4.5 -- inflate64=1.0.0 +- inflate64=1.0.1 - iniconfig=2.0.0 -- ipopt=3.14.16 +- ipopt=3.14.17 - ipykernel=6.29.5 -- ipython=8.30.0 +- ipython=8.31.0 - isoduration=20.11.0 - jedi=0.19.2 -- jinja2=3.1.4 +- jinja2=3.1.5 - joblib=1.4.2 - json5=0.10.0 - jsonpointer=3.0.0 @@ -179,10 +181,10 @@ dependencies: - jupyter-lsp=2.2.5 - jupyter_client=8.6.3 - jupyter_core=5.7.2 -- jupyter_events=0.10.0 -- jupyter_server=2.14.2 +- jupyter_events=0.11.0 +- jupyter_server=2.15.0 - jupyter_server_terminals=0.5.3 -- jupyterlab=4.3.3 +- jupyterlab=4.3.4 - jupyterlab_pygments=0.3.0 - jupyterlab_server=2.27.3 - kealib=1.6.0 @@ -202,10 +204,10 @@ dependencies: - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 -- libclang13=19.1.5 +- libclang13=19.1.6 - libcrc32c=1.1.2 - libcurl=8.11.1 -- libdeflate=1.22 +- libdeflate=1.23 - libevent=2.1.12 - libexpat=2.6.4 - libffi=3.4.2 @@ -238,7 +240,7 @@ dependencies: - libkml=1.3.0 - liblapack=3.9.0 - liblzma=5.6.3 -- libmamba=2.0.4 +- libmamba=2.0.5 - libnetcdf=4.9.2 - libogg=1.3.5 - libopenblas=0.3.28 @@ -250,7 +252,7 @@ dependencies: - librttopo=1.1.0 - libsodium=1.0.20 - libsolv=0.7.30 -- libspatialindex=2.0.0 +- libspatialindex=2.1.0 - libspatialite=5.1.0 - libsqlite=3.47.2 - libssh2=1.11.1 @@ -258,7 +260,7 @@ dependencies: - libtiff=4.7.0 - libutf8proc=2.9.0 - libvorbis=1.3.7 -- libwebp-base=1.4.0 +- libwebp-base=1.5.0 - libwinpthread=12.0.0.r4.gg4f2fc60ca - libxcb=1.17.0 - libxml2=2.13.5 @@ -274,7 +276,7 @@ dependencies: - lz4=4.3.3 - lz4-c=1.10.0 - lzo=2.10 -- mamba=2.0.4 +- mamba=2.0.5 - mapclassify=2.8.1 - markdown=3.6 - markdown-it-py=3.0.0 @@ -294,7 +296,7 @@ dependencies: - multivolumefile=0.2.3 - mumps-seq=5.7.3 - munkres=1.1.4 -- nbclient=0.10.1 +- nbclient=0.10.2 - nbconvert-core=7.16.4 - nbformat=5.10.4 - nest-asyncio=1.6.0 @@ -317,9 +319,9 @@ dependencies: - packaging=24.2 - pandas=2.2.2 - pandocfilters=1.5.0 -- panel=1.5.4 +- panel=1.5.5 - pango=1.54.0 -- param=2.1.1 +- param=2.2.0 - parso=0.8.4 - partd=1.4.2 - patsy=1.0.1 @@ -342,6 +344,7 @@ dependencies: - proj=9.5.1 - prometheus_client=0.21.1 - prompt-toolkit=3.0.48 +- protobuf=5.28.2 - psutil=6.1.0 - pthread-stubs=0.4 - pulp=2.7.0 @@ -350,7 +353,7 @@ dependencies: - py7zr=0.22.0 - pyarrow=18.1.0 - pyarrow-core=18.1.0 -- pybcj=1.0.2 +- pybcj=1.0.3 - pycountry=24.6.1 - pycparser=2.22 - pycryptodomex=3.21.0 @@ -407,10 +410,10 @@ dependencies: - send2trash=1.8.3 - setuptools=75.6.0 - shapely=2.0.6 -- simdjson=3.10.1 +- simdjson=3.11.3 - sip=6.7.12 - six=1.17.0 -- smart_open=7.0.5 +- smart_open=7.1.0 - smmap=5.0.0 - snakemake-minimal=7.32.4 - snappy=1.2.1 @@ -429,7 +432,7 @@ dependencies: - texttable=1.7.0 - threadpoolctl=3.5.0 - throttler=1.2.2 -- tiledb=2.26.2 +- tiledb=2.27.0 - tinycss2=1.4.0 - tk=8.6.13 - toml=0.10.2 @@ -451,7 +454,7 @@ dependencies: - unidecode=1.3.8 - uri-template=1.3.0 - uriparser=0.9.8 -- urllib3=2.2.3 +- urllib3=2.3.0 - validators=0.34.0 - vc=14.3 - vc14_runtime=14.42.34433 @@ -468,10 +471,10 @@ dependencies: - xarray=2023.11.0 - xerces-c=3.2.5 - xlrd=2.0.1 -- xorg-libice=1.1.1 -- xorg-libsm=1.2.4 +- xorg-libice=1.1.2 +- xorg-libsm=1.2.5 - xorg-libx11=1.8.10 -- xorg-libxau=1.0.11 +- xorg-libxau=1.0.12 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - xorg-libxpm=3.5.17 @@ -490,11 +493,9 @@ dependencies: - zstd=1.5.6 - pip: - chaospy==4.3.17 - - earth-osm==2.2 - fake-useragent==2.0.3 - googledrivedownloader==0.4 - - highspy==1.8.1 + - highspy==1.9.0 - polars==1.17.1 - - protobuf==5.29.1 - tsam==2.3.6 prefix: C:\Miniconda\envs\pypsa-earth From e3fc578d44d4d2f40b462a10ca96b4f19d2a5296 Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Tue, 24 Dec 2024 15:16:40 +0100 Subject: [PATCH 106/113] Release v0.6.0 (#1264) * Update release notes * Add a clarification to the release guide * Update the config versions * Clarify the release guidance * Restore minor changes --------- Co-authored-by: Davide Fioriti <67809479+davide-f@users.noreply.github.com> --- config.default.yaml | 2 +- config.tutorial.yaml | 2 +- doc/conf.py | 2 +- doc/release_notes.rst | 14 +++++++++++--- test/config.custom.yaml | 2 +- test/config.landlock.yaml | 2 +- test/config.monte_carlo.yaml | 2 +- test/config.sector.yaml | 2 +- test/config.test_myopic.yaml | 2 +- 9 files changed, 19 insertions(+), 11 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 715a8a4a9..ed8554002 100644 --- a/config.default.yaml +++ b/config.default.yaml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: CC0-1.0 -version: 0.5.0 +version: 0.6.0 tutorial: false logging: diff --git a/config.tutorial.yaml b/config.tutorial.yaml index 7ada63032..8f31af9bf 100644 --- a/config.tutorial.yaml +++ b/config.tutorial.yaml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: CC0-1.0 -version: 0.5.0 +version: 0.6.0 tutorial: true diff --git a/doc/conf.py b/doc/conf.py index cf4323277..b8ff6c30c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -65,7 +65,7 @@ copyright = f"{datetime.datetime.today().year}, {author}" # The full version, including alpha/beta/rc tags -release = "0.5.0" +release = "0.6.0" # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" diff --git a/doc/release_notes.rst b/doc/release_notes.rst index cf5b93895..280904488 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,13 +13,21 @@ This part of documentation collects descriptive release notes to capture the mai **New Features and Major Changes** + +**Minor Changes and bug-fixing** + + +PyPSA-Earth 0.6.0 +================= + +**New Features and Major Changes (24th December 2024)** + * Include option in the config to allow for custom airport data `PR #1241 `__ * Added Dev Containers and docker as an option to get started with pypsa-earth `PR #1228 `__ * Add a list of PyPSA-Earth applications in academic and industrial projects `PR #1255 `__ - * Computational improvements of build_osm_network `PR #845 `__ * Boost computational performances of set_lines_ids with cKDTree by scipy `PR #806 `__ @@ -537,13 +545,13 @@ Release Process * Make sure thah pinned versions of the environments ``*-pinned.yaml`` in ``envs`` folder are up-to-date. -* Update version number in ``doc/conf.py`` and ``*config.*.yaml``. +* Update version number in ``doc/conf.py``, ``default.config.yaml``, ``tutorial.config.yaml`` and ``test/config.*.yaml``. * Open, review and merge pull request for branch ``release-v0.x.x``. Make sure to close issues and PRs or the release milestone with it (e.g. closes #X). Run ``pre-commit run --all`` locally and fix any issues. -* Tag a release on Github via ``git tag v0.x.x``, ``git push``, ``git push --tags``. Include release notes in the tag message. +* Update and checkout your local `main` and tag a release with ``git tag v0.x.x``, ``git push``, ``git push --tags``. Include release notes in the tag message using Github UI. * Upload code to `zenodo code repository `_ with `GPLv3 license `_. diff --git a/test/config.custom.yaml b/test/config.custom.yaml index a596a932d..5cd36f44f 100644 --- a/test/config.custom.yaml +++ b/test/config.custom.yaml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 ### CHANGES TO CONFIG.TUTORIAL.YAML ### -version: 0.5.0 +version: 0.6.0 run: name: "custom" diff --git a/test/config.landlock.yaml b/test/config.landlock.yaml index 913211f29..fc267e829 100644 --- a/test/config.landlock.yaml +++ b/test/config.landlock.yaml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 ### CHANGES TO CONFIG.TUTORIAL.YAML ### -version: 0.5.0 +version: 0.6.0 countries: ["BW"] diff --git a/test/config.monte_carlo.yaml b/test/config.monte_carlo.yaml index 034dd51cd..c35dde51f 100644 --- a/test/config.monte_carlo.yaml +++ b/test/config.monte_carlo.yaml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 ### CHANGES TO CONFIG.TUTORIAL.YAML ### -version: 0.5.0 +version: 0.6.0 monte_carlo: options: diff --git a/test/config.sector.yaml b/test/config.sector.yaml index abc250e0c..670344b0a 100644 --- a/test/config.sector.yaml +++ b/test/config.sector.yaml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later -version: 0.5.0 +version: 0.6.0 tutorial: true run: diff --git a/test/config.test_myopic.yaml b/test/config.test_myopic.yaml index 05f3c71a1..382def55f 100644 --- a/test/config.test_myopic.yaml +++ b/test/config.test_myopic.yaml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-or-later -version: 0.5.0 +version: 0.6.0 logging_level: INFO tutorial: true From 4993f9b9b551a156684ed404d8a618cf5f172140 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Dec 2024 19:18:17 +0100 Subject: [PATCH 107/113] Bump the github-actions group with 2 updates (#1266) Bumps the github-actions group with 2 updates: [actions/checkout](https://github.com/actions/checkout) and [docker/login-action](https://github.com/docker/login-action). Updates `actions/checkout` from 1 to 4 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v1...v4) Updates `docker/login-action` from 2 to 3 - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/devcontainer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 85de566fe..ec5b4cdf0 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -14,10 +14,10 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ secrets.REGISTRY_USER }} From c551b82e5de7ed298c86a4a28834b749ffc46b6b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:55:56 +0100 Subject: [PATCH 108/113] docs(contributor): contrib-readme-action has updated readme (#1273) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- README.md | 708 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 391 insertions(+), 317 deletions(-) diff --git a/README.md b/README.md index 34b998185..6de82672e 100644 --- a/README.md +++ b/README.md @@ -187,322 +187,396 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
- - SermishaNarayana -
- Null - - 		- - davide-f -
- Davide-f - - 		- - ekatef -
- Ekaterina - - 		- - pz-max -
- Max Parzen - - 		- - DeniseGiub -
- DeniseGiub - - 		- - yerbol-akhmetov -
- Yerbol Akhmetov - -
- - GbotemiB -
- Emmanuel Bolarinwa - - 		- - mnm-matin -
- Mnm-matin - - 		- - hazemakhalek -
- Hazem - - 		- - energyLS -
- EnergyLS - - 		- - Tomkourou -
- Thomas Kouroughli - - 		- - GridGrapher -
- GridGrapher - -
- - martacki -
- Martha Frysztacki - - 		- - finozzifa -
- Finozzifa - - 		- - Emre-Yorat89 -
- Emre_Yorat - - 		- - virio-andreyana -
- Null - - 		- - giacfalk -
- Giacomo Falchetta - - 		- - Ekaterina-Vo -
- Ekaterina-Vo - -
- - cpschau -
- Cschau - - 		- - euronion -
- Euronion - - 		- - AnasAlgarei -
- AnasAlgarei - - 		- - LukasFrankenQ -
- Lukas Franken - - 		- - Tooblippe -
- Tobias - - 		- - doneachh -
- Anton Achhammer - -
- - koen-vg -
- Koen Van Greevenbroek - - 		- - carlosfv92 -
- Carlos Fernandez - - 		- - koen-vg -
- Koen Van Greevenbroek - - 		- - asolavi -
- Null - - 		- - Netotse -
- Null - -
- - pitmonticone -
- Pietro Monticone - - 		- - siddharth-krishna -
- Siddharth Krishna - -
- - squoilin -
- Sylvain Quoilin - - 		- - juli-a-ko -
- Juli-a-ko - - 		- - ollie-bell -
- Null - - 		- - rsparks3 -
- Ryan - - 		- - stephenjlee -
- Stephen J Lee - - 		- - kma33 -
- Katherine M. Antonio - -
- - jessLryan -
- Jess - - 		- - jarry7 -
- Jarrad Wright - - 		- - HanaElattar -
- HanaElattar - - 		- - FabianHofmann -
- Fabian Hofmann - - 		- - EmreYorat -
- EmreYorat - - 		- - AndreCNF -
- André Cristóvão Neves Ferreira - -
- - AlexanderMeisinger -
- Null - -
+ + FabianHofmann +
+ Fabian Hofmann + + 		+ + fneum +
+ Fabian Neumann + + 		+ + ekatef +
+ Ekaterina + + 		+ + euronion +
+ Euronion + + 		+ + Justus-coded +
+ Justus Ilemobayo + + 		+ + mnm-matin +
+ Mnm-matin + +
+ + martacki +
+ Martha Frysztacki + + 		+ + LukasFrankenQ +
+ Lukas Franken + + 		+ + pz-max +
+ Max Parzen + + 		+ + davide-f +
+ Davide-f + + 		+ + koen-vg +
+ Koen Van Greevenbroek + + 		+ + Eddy-JV +
+ Eddy Jalbout + +
+ + hazemakhalek +
+ Hazem + + 		+ + energyLS +
+ EnergyLS + + 		+ + AnasAlgarei +
+ AnasAlgarei + + 		+ + yerbol-akhmetov +
+ Yerbol Akhmetov + + 		+ + GbotemiB +
+ Emmanuel Bolarinwa + + 		+ + DeniseGiub +
+ DeniseGiub + +
+ + doneachh +
+ Anton Achhammer + + 		+ + Tomkourou +
+ Thomas Kouroughli + + 		+ + finozzifa +
+ Finozzifa + + 		+ + GridGrapher +
+ GridGrapher + + 		+ + drifter089 +
+ Akshat Mittal + + 		+ + glenkiely-ieg +
+ glenkiely-ieg + +
+ + cpschau +
+ Cschau + + 		+ + Emre-Yorat89 +
+ Emre_Yorat + + 		+ + virio-andreyana +
+ Null + + 		+ + giacfalk +
+ Giacomo Falchetta + + 		+ + Ekaterina-Vo +
+ Ekaterina-Vo + + 		+ + lkstrp +
+ Lukas Trippe + +
+ + Tooblippe +
+ Tobias + + 		+ + arizeosalac +
+ zeosalac + + 		+ + danielelerede-oet +
+ danielelerede-oet + + 		+ + carlosfv92 +
+ Carlos Fernandez + + 		+ + rajesh-ieg +
+ rajesh-ieg + + 		+ + asolavi +
+ Null + +
+ + stephenjlee +
+ Stephen J Lee + + 		+ + rsparks3 +
+ Ryan + + 		+ + ollie-bell +
+ Null + + 		+ + juli-a-ko +
+ Juli-a-ko + + 		+ + squoilin +
+ Sylvain Quoilin + + 		+ + siddharth-krishna +
+ Siddharth Krishna + +
+ + SermishaNarayana +
+ Null + + 		+ + pitmonticone +
+ Pietro Monticone + + 		+ + Netotse +
+ Null + + 		+ + milyas009 +
+ Muhammad Ilyas + + 		+ + kma33 +
+ Katherine M. Antonio + + 		+ + jessLryan +
+ Jess + +
+ + jarry7 +
+ Jarrad Wright + + 		+ + HanaElattar +
+ HanaElattar + + 		+ + EmreYorat +
+ EmreYorat + + 		+ + AndreCNF +
+ André Cristóvão Neves Ferreira + + 		+ + AlexanderMeisinger +
+ Null + +
From 0a09bcaf738db25723cf1a4af3458c7eeb14bdec Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sat, 4 Jan 2025 17:55:45 +0500 Subject: [PATCH 109/113] do not compute powerplants from ppm if replace option is used --- scripts/build_powerplants.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 4bf22e524..b1719108d 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -337,13 +337,16 @@ def replace_natural_gas_technology(df: pd.DataFrame): else: config["main_query"] = "" - ppl = ( - pm.powerplants(from_url=False, update=True, config_update=config) - .powerplant.fill_missing_decommissioning_years() - .query('Fueltype not in ["Solar", "Wind"] and Country in @countries_names') - .powerplant.convert_country_to_alpha2() - .pipe(replace_natural_gas_technology) - ) + if snakemake.config["electricity"]["custom_powerplants"] != "replace": + ppl = ( + pm.powerplants(from_url=False, update=True, config_update=config) + .powerplant.fill_missing_decommissioning_years() + .query('Fueltype not in ["Solar", "Wind"] and Country in @countries_names') + .powerplant.convert_country_to_alpha2() + .pipe(replace_natural_gas_technology) + ) + else: + ppl = pd.DataFrame() ppl = add_custom_powerplants( ppl, snakemake.input, snakemake.config From c3a1eb19802361e91f2b1efb6eb851a9eca1a52a Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Sat, 4 Jan 2025 19:16:42 +0500 Subject: [PATCH 110/113] Add release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 280904488..9faf38931 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -42,6 +42,8 @@ PyPSA-Earth 0.6.0 * Update BW, NG and BJ tutorial databundles to include gadm-like sources from geoboundaries `PR #1257 `__ +* Prevent computation of powerplantmatching if replace option is selected for custom_powerplants `PR #1281 `__ + PyPSA-Earth 0.5.0 ================= From 5b9ac3038daa4fe3bda654bb9b43c305f98506f0 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Sat, 4 Jan 2025 15:46:44 +0100 Subject: [PATCH 111/113] Revise export path of unsd data (#1279) --- Snakefile | 3 ++- scripts/build_base_industry_totals.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index a0e1a3a75..5002fe2c5 100644 --- a/Snakefile +++ b/Snakefile @@ -1358,6 +1358,7 @@ rule build_base_energy_totals: unsd_paths="data/demand/unsd/paths/Energy_Statistics_Database.xlsx", output: energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv", + unsd_export_path=directory("data/demand/unsd/data/"), script: "scripts/build_base_energy_totals.py" @@ -1877,7 +1878,7 @@ rule build_base_industry_totals: #default data input: #os.path.dirname(snakemake.input["transactions_path"]) + "/demand/unsd/data/" #industrial_production_per_country="data/industrial_production_per_country.csv", - unsd_path="data/demand/unsd/data/", + unsd_export_path="data/demand/unsd/data/", energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv", transactions_path="data/unsd_transactions.csv", output: diff --git a/scripts/build_base_industry_totals.py b/scripts/build_base_industry_totals.py index 1e5eda9a8..977f95ea8 100644 --- a/scripts/build_base_industry_totals.py +++ b/scripts/build_base_industry_totals.py @@ -117,7 +117,7 @@ def create_industry_base_totals(df): renaming_dit = transaction.set_index("Transaction")["clean_name"].to_dict() clean_industry_list = list(transaction.clean_name.unique()) - unsd_path = snakemake.input.unsd_path + unsd_path = snakemake.input.unsd_export_path # Get the files from the path provided in the OP all_files = list(Path(unsd_path).glob("*.txt")) From 395aaecd98112b82ff46c72f75a19b1ecaab36de Mon Sep 17 00:00:00 2001 From: Davide Fioriti Date: Sat, 4 Jan 2025 17:04:21 +0100 Subject: [PATCH 112/113] Move release note and drop DateMothball column from custom_powerplants --- data/custom_powerplants.csv | 2 +- doc/release_notes.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/custom_powerplants.csv b/data/custom_powerplants.csv index fb83a5ff4..d81c32bca 100644 --- a/data/custom_powerplants.csv +++ b/data/custom_powerplants.csv @@ -1 +1 @@ -Name,Fueltype,Technology,Set,Country,Capacity,Efficiency,Duration,Volume_Mm3,DamHeight_m,StorageCapacity_MWh,DateIn,DateRetrofit,DateMothball,DateOut,lat,lon,EIC,projectID,bus +Name,Fueltype,Technology,Set,Country,Capacity,Efficiency,Duration,Volume_Mm3,DamHeight_m,StorageCapacity_MWh,DateIn,DateRetrofit,DateOut,lat,lon,EIC,projectID,bus diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9faf38931..80f8f7410 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -16,6 +16,8 @@ This part of documentation collects descriptive release notes to capture the mai **Minor Changes and bug-fixing** +* Prevent computation of powerplantmatching if replace option is selected for custom_powerplants `PR #1281 `__ + PyPSA-Earth 0.6.0 ================= @@ -42,8 +44,6 @@ PyPSA-Earth 0.6.0 * Update BW, NG and BJ tutorial databundles to include gadm-like sources from geoboundaries `PR #1257 `__ -* Prevent computation of powerplantmatching if replace option is selected for custom_powerplants `PR #1281 `__ - PyPSA-Earth 0.5.0 ================= From ef525ee237b27916caf9a94a2d2157d25bd962f3 Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Sun, 5 Jan 2025 00:55:48 +0100 Subject: [PATCH 113/113] Use username for contributors (#1282) * Use username for contributors * Workflow dispatch bypass contributor workflow --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9ebd17f4d..8cf4c6ce4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,9 +7,11 @@ jobs: contrib-readme-job: runs-on: ubuntu-latest name: A job to automate contrib in readme - if: ${{ github.repository_owner == 'pypsa-meets-earth' && github.ref == 'refs/heads/main'}} + if: ${{ github.event_name == 'workflow_dispatch' || (github.repository_owner == 'pypsa-meets-earth' && github.ref == 'refs/heads/main')}} steps: - name: Contribute List uses: akhilmhdh/contributors-readme-action@v2.3.10 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + use_username: true