Calibration run 2023/2024

config/scenarios.form.yaml

@@ -118,4 +118,75 @@ main-mds-vanilla-with-grids:
     storage_units: ["li-ion battery", "iron-air battery"]
     H2_network: true
     gas_network: true
-    H2_retrofit: true
+    H2_retrofit: true
+
+main-calibration-run:
+  foresight: overnight
+  scenario:
+    ll:
+    - v1.0
+    planning_horizons:
+    - 2023
+  enable:
+    final_adjustment: false
+  snapshots:
+    start: "2023-01-01"
+    end: "2024-01-01"
+  co2_budget:
+    2023: 0.378 
+    # 63% from https://commission.europa.eu/news/climate-report-shows-largest-annual-drop-eu-greenhouse-gas-emissions-decades-2024-11-05_en
+    # 60% factored in emission included sector from co2_totals_sector.csv (interpolation between 2015, 2030)
+    # 0.63 * 0.6 = 0.378
+  electricity:
+    max_hours:
+      li-ion battery: [1, 2, 4, 8]
+      vanadium: [10]
+      lair: [12]
+      pair: [24]
+      H2: [168]
+      iron-air battery: [100]
+    powerplants_filter: DateOut >= 2023 and not Fueltype.isin(['Nuclear', 'Hard Coal', 'Lignite'])
+    custom_powerplants: DateIn <= 2023 and DateOut >= 2023
+  atlite:
+    default_cutout: europe-2023-sarah3-era5
+  renewable:
+    onwind:
+      cutout: europe-2023-sarah3-era5
+      #correction_factor: 0.93
+    offwind-ac:
+      cutout: europe-2023-sarah3-era5
+    offwind-dc:
+      cutout: europe-2023-sarah3-era5
+    offwind-float:
+      cutout: europe-2023-sarah3-era5
+    solar:
+      cutout: europe-2023-sarah3-era5
+      correction_factor: 0.854337 #Using ERA5 data requires correction factor for solar.
+    solar-hsat:
+      cutout: europe-2023-sarah3-era5
+    hydro:
+      cutout: europe-2023-sarah3-era5
+      #eia_norm_year: 2021 #The last year available from EIA for hydro is 2021. If not activated, the median value is chosen
+  biomass:
+    year: 2020
+    share_unsustainable_use_retained:
+      2023: 0.8
+    share_sustainable_potential_available:
+      2023: 0.2
+  sector:
+    district_heating:
+      progress:
+        2023: 0.15 #Rough estimate. 70M/448M population uses district heating https://iifiir.org/en/news/state-of-play-of-district-heating-and-cooling-in-europe
+    land_transport_fuel_cell_share:
+      2023: 0
+    land_transport_electric_share:
+      2023: 0.02 #https://ec.europa.eu/eurostat/web/products-eurostat-news/w/ddn-20240802-1
+    land_transport_ice_share:
+      2023: 0.98 # 1.00 - 0.02
+    co2_sequestration_potential:
+      2023: 0
+    stores: []
+    storage_units: ["li-ion battery"]
+    methanation: false
+  costs:
+    year: 2020

rules/validate.smk

@@ -11,6 +11,16 @@ CROSS_BORDER_PLOTS = ["trade_time_series", "cross_border_bar"]
 PRICES_PLOTS = ["price_bar", "price_line"]
 
 
+if config["foresight"] == "myopic" or config["foresight"] == "overnight":
+    NETWORK_VALIDATE_INPUT = RESULTS + "postnetworks/base_s_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",
+    NETWORK_VALIDATE_OUTPUT = "base_s_{clusters}_elec_l{ll}_{opts}_{sector_opts}_{planning_horizons}"
+elif config["foresight"] == "perfect":
+    NETWORK_VALIDATE_INPUT = RESULTS + "postnetworks/base_s_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc",
+    NETWORK_VALIDATE_OUTPUT = "base_s_{{clusters}}_elec_l{ll}_{opts}_{sector_opts}_brownfield_all_years"
+else:
+    NETWORK_VALIDATE_INPUT = RESULTS + "networks/base_s_{clusters}_elec_l{ll}_{opts}.nc"
+    NETWORK_VALIDATE_OUTPUT = "base_s_{clusters}_elec_l{ll}_{opts}"
+
 rule build_electricity_production:
     """
     This rule builds the electricity production for each country and technology from ENTSO-E data.
@@ -69,16 +79,16 @@ rule build_electricity_prices:
 
 rule plot_validation_electricity_production:
     input:
-        network=RESULTS + "networks/base_s_{clusters}_elec_l{ll}_{opts}.nc",
+        network=NETWORK_VALIDATE_INPUT,
         electricity_production=resources("historical_electricity_production.csv"),
     output:
         **{
             plot: RESULTS
-            + f"figures/validation_{plot}_base_s_{{clusters}}_elec_l{{ll}}_{{opts}}.pdf"
+            + f"figures/validation_{plot}_" + NETWORK_VALIDATE_OUTPUT + ".pdf"
             for plot in PRODUCTION_PLOTS
         },
         plots_touch=RESULTS
-        + "figures/.validation_production_plots_base_s_{clusters}_elec_l{ll}_{opts}",
+        + "figures/.validation_production_plots_"+ NETWORK_VALIDATE_OUTPUT,
     script:
         "../scripts/plot_validation_electricity_production.py"
 
@@ -87,31 +97,31 @@ rule plot_validation_cross_border_flows:
     params:
         countries=config_provider("countries"),
     input:
-        network=RESULTS + "networks/base_s_{clusters}_elec_l{ll}_{opts}.nc",
+        network=NETWORK_VALIDATE_INPUT,
         cross_border_flows=resources("historical_cross_border_flows.csv"),
     output:
         **{
             plot: RESULTS
-            + f"figures/validation_{plot}_base_s_{{clusters}}_elec_l{{ll}}_{{opts}}.pdf"
+            + f"figures/validation_{plot}_" + NETWORK_VALIDATE_OUTPUT + ".pdf"
             for plot in CROSS_BORDER_PLOTS
         },
         plots_touch=RESULTS
-        + "figures/.validation_cross_border_plots_base_s_{clusters}_elec_l{ll}_{opts}",
+        + "figures/.validation_cross_border_plots_" + NETWORK_VALIDATE_OUTPUT,
     script:
         "../scripts/plot_validation_cross_border_flows.py"
 
 
 rule plot_validation_electricity_prices:
     input:
-        network=RESULTS + "networks/base_s_{clusters}_elec_l{ll}_{opts}.nc",
+        network=NETWORK_VALIDATE_INPUT,
         electricity_prices=resources("historical_electricity_prices.csv"),
     output:
         **{
             plot: RESULTS
-            + f"figures/validation_{plot}_base_s_{{clusters}}_elec_l{{ll}}_{{opts}}.pdf"
+            + f"figures/validation_{plot}_" + NETWORK_VALIDATE_OUTPUT + ".pdf"
             for plot in PRICES_PLOTS
         },
         plots_touch=RESULTS
-        + "figures/.validation_prices_plots_base_s_{clusters}_elec_l{ll}_{opts}",
+        + "figures/.validation_prices_plots_" + NETWORK_VALIDATE_OUTPUT,
     script:
         "../scripts/plot_validation_electricity_prices.py"

scripts/build_hydro_profile.py

@@ -207,9 +207,9 @@ def approximate_missing_eia_stats(eia_stats, runoff_fn, countries):
     norm_year = config_hydro.get("eia_norm_year")
     missing_years = contained_years.difference(eia_stats.index)
     if norm_year:
-        eia_stats.loc[contained_years] = eia_stats.loc[norm_year]
+        eia_stats.loc[contained_years[0]] = eia_stats.loc[norm_year]
     elif missing_years.any():
-        eia_stats.loc[missing_years] = eia_stats.median()
+        eia_stats.loc[missing_years[0]] = eia_stats.median()
 
     inflow = cutout.runoff(
         shapes=country_shapes,

scripts/plot_validation_cross_border_flows.py

@@ -205,6 +205,10 @@ def cross_border_bar(countries, data):
     if len(historic.index) > len(n.snapshots):
         historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots]
 
+    # Set the historic date based on the snapshot year
+    if historic.index.year.unique()[0] != n.snapshots.year.unique()[0]:
+        historic.index = historic.index.map(lambda x: x.replace(year=n.snapshots.year.unique()[0]))
+
     # Preparing network data to be shaped similar to ENTSOE datastructure
     optimized_links = n.links_t.p0.rename(
         columns=dict(n.links.bus0.str[:2] + " - " + n.links.bus1.str[:2])

scripts/plot_validation_electricity_prices.py

@@ -38,7 +38,14 @@
     if len(historic.index) > len(n.snapshots):
         historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots]
 
-    optimized = n.buses_t.marginal_price.groupby(n.buses.country, axis=1).mean()
+    optimized = n.buses_t.marginal_price.T.groupby(n.buses.country).mean().T
+
+    # Remove all carriers originated not from a country
+    optimized = optimized.loc[:, optimized.columns != '']
+
+        # Set the historic date based on the snapshot year
+    if historic.index.year.unique()[0] != n.snapshots.year.unique()[0]:
+        optimized.index = optimized.index.map(lambda x: x.replace(year=historic.index.year.unique()[0]))
 
     data = pd.concat([historic, optimized], keys=["Historic", "Optimized"], axis=1)
     data.columns.names = ["Kind", "Country"]

scripts/plot_validation_electricity_production.py

@@ -20,6 +20,9 @@
     "Combined-Cycle Gas": "Gas",
     "Reservoir & Dam": "Hydro",
     "Pumped Hydro Storage": "Hydro",
+    "Solar":"Solar",
+    "solar-hsat":"Solar",
+    "solar rooftop":"Solar",
 }
 
 
@@ -37,7 +40,7 @@
     set_scenario_config(snakemake)
 
     n = pypsa.Network(snakemake.input.network)
-    n.loads.carrier = "load"
+    #n.loads.carrier = "load"
 
     historic = pd.read_csv(
         snakemake.input.electricity_production,
@@ -58,23 +61,39 @@
     colors["Offshore Wind"] = colors["Offshore Wind (AC)"]
     colors["Gas"] = colors["Combined-Cycle Gas"]
     colors["Hydro"] = colors["Reservoir & Dam"]
+    colors["geothermal"] = '#ba91b1'
+    colors["biomass"] = '#baa741'
     colors["Other"] = "lightgray"
 
     if len(historic.index) > len(n.snapshots):
         historic = historic.resample(n.snapshots.inferred_freq).mean().loc[n.snapshots]
 
-    optimized = n.statistics.dispatch(
+    # Set the historic date based on the snapshot year
+    historic_year = historic.index.year.unique()[0]
+    if historic_year != n.snapshots.year.unique()[0]:
+        historic.index = historic.index.map(lambda x: x.replace(year=n.snapshots.year.unique()[0]))
+
+    optimized = n.statistics.energy_balance(
         groupby=get_bus_and_carrier, aggregate_time=False
     ).T
-    optimized = optimized[["Generator", "StorageUnit"]].droplevel(0, axis=1)
+    optimized = optimized[["Generator","StorageUnit","Link"]].droplevel(0, axis=1)
     optimized = optimized.rename(columns=n.buses.country, level=0)
     optimized = optimized.rename(columns=carrier_groups, level=1)
     optimized = optimized.T.groupby(level=[0, 1]).sum().T
 
+    # Remove all carriers originated not from a country
+    optimized = optimized.loc[:, optimized.columns.get_level_values(0) != '']
+
+    # Compare carrier where historical data are available
+    optimized = optimized.loc[:, optimized.columns.get_level_values(1).isin(historic.columns.get_level_values(1))]
+
     data = pd.concat([historic, optimized], keys=["Historic", "Optimized"], axis=1)
     data.columns.names = ["Kind", "Country", "Carrier"]
     data = data.mul(n.snapshot_weightings.generators, axis=0)
 
+    # revert back datetime according to historical year
+    data.index = data.index.map(lambda x: x.replace(year=historic_year))
+
     # total production per carrier
     fig, ax = plt.subplots(figsize=(6, 6))
 

scripts/prepare_sector_network.py

@@ -2170,7 +2170,7 @@ def add_fuel_cell_cars(n, p_set, fuel_cell_share, temperature):
         suffix=" land transport fuel cell",
         bus=spatial.h2.nodes,
         carrier="land transport fuel cell",
-        p_set=profile,
+        p_set=profile.loc[n.snapshots],
     )
 
 
@@ -2209,7 +2209,7 @@ def add_ice_cars(n, p_set, ice_share, temperature):
         spatial.oil.land_transport,
         bus=spatial.oil.land_transport,
         carrier="land transport oil",
-        p_set=profile,
+        p_set=profile.loc[n.snapshots],
     )
 
     n.add(