From 7f93dd07b6f85048a1bad3952cd2a96004e72ba7 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 10 Dec 2024 19:46:59 +0100 Subject: [PATCH 1/4] Put country-specific data into resources folder --- Snakefile | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/Snakefile b/Snakefile index 767833df7..086a4421a 100644 --- a/Snakefile +++ b/Snakefile @@ -1004,7 +1004,7 @@ rule prepare_ports: params: custom_export=config["custom_data"]["export_ports"], output: - ports="data/ports.csv", # TODO move from data to resources + ports="resources/" + SECDIR + "ports.csv", export_ports="resources/" + SECDIR + "export_ports.csv", script: "scripts/prepare_ports.py" @@ -1014,14 +1014,14 @@ rule prepare_airports: params: airport_sizing_factor=config["sector"]["airport_sizing_factor"], output: - ports="data/airports.csv", # TODO move from data to resources + ports="resources/" + SECDIR + "airports.csv", script: "scripts/prepare_airports.py" rule prepare_urban_percent: output: - urban_percent="data/urban_percent.csv", # TODO move from data to resources + urban_percent="resources/" + SECDIR + "urban_percent.csv", script: "scripts/prepare_urban_percent.py" @@ -1094,9 +1094,11 @@ rule prepare_sector_network: industrial_demand="resources/" + SECDIR + "demand/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv", - energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", - airports="data/airports.csv", - ports="data/ports.csv", + energy_totals="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", + airports="resources/" + SECDIR + "airports.csv", + ports="resources/" + SECDIR + "ports.csv", heat_demand="resources/" + SECDIR + "demand/heat/heat_demand_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv", @@ -1200,7 +1202,9 @@ rule override_respot: }, overrides="data/override_component_attrs", network="networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", + energy_totals="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", output: RESDIR + "prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_presec.nc", @@ -1211,7 +1215,9 @@ rule override_respot: rule prepare_transport_data: input: network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", - energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", + energy_totals_name="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", traffic_data_KFZ="data/emobility/KFZ__count", traffic_data_Pkw="data/emobility/Pkw__count", transport_name="resources/" + SECDIR + "transport_data.csv", @@ -1295,7 +1301,9 @@ rule build_cop_profiles: rule prepare_heat_data: input: network="networks/" + RDIR + "elec_s{simpl}_{clusters}.nc", - energy_totals_name="data/energy_totals_{demand}_{planning_horizons}.csv", + energy_totals_name="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", clustered_pop_layout="resources/" + SECDIR + "population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv", @@ -1348,7 +1356,7 @@ rule build_base_energy_totals: input: unsd_paths="data/demand/unsd/paths/Energy_Statistics_Database.xlsx", output: - energy_totals_base="data/energy_totals_base.csv", + energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv", script: "scripts/build_base_energy_totals.py" @@ -1359,13 +1367,15 @@ rule prepare_energy_totals: base_year=config["demand_data"]["base_year"], sector_options=config["sector"], input: - unsd_paths="data/energy_totals_base.csv", + unsd_paths="resources/" + SECDIR + "energy_totals_base.csv", efficiency_gains_cagr="data/demand/efficiency_gains_cagr.csv", growth_factors_cagr="data/demand/growth_factors_cagr.csv", district_heating="data/demand/district_heating.csv", fuel_shares="data/demand/fuel_shares.csv", output: - energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv", + energy_totals="resources/" + + SECDIR + + "energy_totals_{demand}_{planning_horizons}.csv", script: "scripts/prepare_energy_totals.py" @@ -1419,7 +1429,7 @@ rule build_population_layouts: planning_horizons=config["scenario"]["planning_horizons"][0], input: nuts3_shapes="resources/" + RDIR + "shapes/gadm_shapes.geojson", - urban_percent="data/urban_percent.csv", + urban_percent="resources/" + SECDIR + "urban_percent.csv", cutout="cutouts/" + CDIR + [c["cutout"] for _, c in config["renewable"].items()][0] @@ -1866,7 +1876,7 @@ rule build_base_industry_totals: #default data input: #industrial_production_per_country="data/industrial_production_per_country.csv", #unsd_path="data/demand/unsd/data/", - energy_totals_base="data/energy_totals_base.csv", + energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv", transactions_path="data/unsd_transactions.csv", output: base_industry_totals="resources/" From 26eafdb54e0e53b1b5f08cf3600e4ed37a677209 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 10 Dec 2024 19:47:33 +0100 Subject: [PATCH 2/4] Replace a benchmark path --- scripts/build_base_industry_totals.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_base_industry_totals.py b/scripts/build_base_industry_totals.py index eef58f10a..46736c61c 100644 --- a/scripts/build_base_industry_totals.py +++ b/scripts/build_base_industry_totals.py @@ -117,8 +117,9 @@ def create_industry_base_totals(df): renaming_dit = transaction.set_index("Transaction")["clean_name"].to_dict() clean_industry_list = list(transaction.clean_name.unique()) + # TODO Check with the sector-coupled modelling experts unsd_path = ( - os.path.dirname(snakemake.input["energy_totals_base"]) + "/demand/unsd/data/" + os.path.dirname(snakemake.input["transactions_path"]) + "/demand/unsd/data/" ) # Get the files from the path provided in the OP From f5a957eb70d0ec53277939f5243a36d62ad5a9d5 Mon Sep 17 00:00:00 2001 From: ekatef Date: Wed, 11 Dec 2024 15:39:13 +0100 Subject: [PATCH 3/4] Replace path hardcoding in script --- scripts/prepare_energy_totals.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py index 708420b27..9e473de5e 100644 --- a/scripts/prepare_energy_totals.py +++ b/scripts/prepare_energy_totals.py @@ -43,8 +43,8 @@ def calculate_end_values(df): snakemake = mock_snakemake( "prepare_energy_totals", simpl="", - clusters=32, - demand="EG", + clusters=10, + demand="AB", planning_horizons=2030, ) @@ -53,9 +53,7 @@ def calculate_end_values(df): investment_year = int(snakemake.wildcards.planning_horizons) demand_sc = snakemake.wildcards.demand # loading the demand scenrario wildcard - base_energy_totals = read_csv_nafix( - os.path.join(BASE_DIR, "data/energy_totals_base.csv"), index_col=0 - ) + base_energy_totals = read_csv_nafix(snakemake.input.unsd_paths, index_col=0) growth_factors_cagr = read_csv_nafix( snakemake.input.growth_factors_cagr, index_col=0 ) From d566e988a93a881dc24fb53aa5312f2ef8ce7db3 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Dec 2024 10:03:58 +0100 Subject: [PATCH 4/4] Put the path into snakemake inputs --- Snakefile | 3 ++- scripts/build_base_industry_totals.py | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Snakefile b/Snakefile index 41d8952fc..a0e1a3a75 100644 --- a/Snakefile +++ b/Snakefile @@ -1875,8 +1875,9 @@ rule build_base_industry_totals: #default data countries=config["countries"], other_industries=config["demand_data"]["other_industries"], input: + #os.path.dirname(snakemake.input["transactions_path"]) + "/demand/unsd/data/" #industrial_production_per_country="data/industrial_production_per_country.csv", - #unsd_path="data/demand/unsd/data/", + unsd_path="data/demand/unsd/data/", energy_totals_base="resources/" + SECDIR + "energy_totals_base.csv", transactions_path="data/unsd_transactions.csv", output: diff --git a/scripts/build_base_industry_totals.py b/scripts/build_base_industry_totals.py index 4c33aaaad..1e5eda9a8 100644 --- a/scripts/build_base_industry_totals.py +++ b/scripts/build_base_industry_totals.py @@ -117,10 +117,7 @@ def create_industry_base_totals(df): renaming_dit = transaction.set_index("Transaction")["clean_name"].to_dict() clean_industry_list = list(transaction.clean_name.unique()) - # TODO Check with the sector-coupled modelling experts - unsd_path = ( - os.path.dirname(snakemake.input["transactions_path"]) + "/demand/unsd/data/" - ) + unsd_path = snakemake.input.unsd_path # Get the files from the path provided in the OP all_files = list(Path(unsd_path).glob("*.txt"))