Merge pull request #318 from pypsa-meets-earth/multiyear

Add myopic optimization

Snakefile

@@ -522,37 +522,39 @@ rule copy_config:
         "scripts/copy_config.py"
 
 
-rule solve_network:
-    input:
-        overrides="data/override_component_attrs",
-        # network=RDIR
-        # + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc",
-        network=RDIR
-        + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
-        costs=CDIR + "costs_{planning_horizons}.csv",
-        configs=SDIR + "/configs/config.yaml",  # included to trigger copy_config rule
-    output:
-        RDIR
-        + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
-    shadow:
-        "shallow"
-    log:
-        solver=RDIR
-        + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log",
-        python=RDIR
-        + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log",
-        memory=RDIR
-        + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log",
-    threads: 25
-    resources:
-        mem_mb=config["solving"]["mem"],
-    benchmark:
-        (
+if config["foresight"] == "overnight":
+
+    rule solve_network:
+        input:
+            overrides="data/override_component_attrs",
+            # network=RDIR
+            # + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}.nc",
+            network=RDIR
+            + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+            costs=CDIR + "costs_{planning_horizons}.csv",
+            configs=SDIR + "/configs/config.yaml",  # included to trigger copy_config rule
+        output:
             RDIR
-            + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export"
-        )
-    script:
-        "scripts/solve_network.py"
+            + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+        shadow:
+            "shallow"
+        log:
+            solver=RDIR
+            + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log",
+            python=RDIR
+            + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log",
+            memory=RDIR
+            + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log",
+        threads: 25
+        resources:
+            mem_mb=config["solving"]["mem"],
+        benchmark:
+            (
+                RDIR
+                + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export"
+            )
+        script:
+            "scripts/solve_network.py"
 
 
 rule make_summary:
@@ -691,6 +693,8 @@ rule run_test:
 
         shell("cp test/config.test1.yaml config.yaml")
         shell("snakemake --cores all solve_all_networks --forceall")
+        shell("cp test/config.test_myopic.yaml config.yaml")
+        shell("snakemake --cores all solve_all_networks_myopic --forceall")
 
 
 
@@ -771,3 +775,182 @@ rule build_industry_demand:  #default data
         "benchmarks/industrial_energy_demand_per_node_elec_s{simpl}_{clusters}_{planning_horizons}_{demand}.csv"
     script:
         "scripts/build_industry_demand.py"
+
+
+rule build_existing_heating_distribution:
+    params:
+        baseyear=config["scenario"]["planning_horizons"][0],
+        sector=config["sector"],
+        existing_capacities=config["existing_capacities"],
+    input:
+        existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
+        clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
+        clustered_pop_energy_layout="resources/demand/heat/nodal_energy_heat_totals_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv",  #"resources/population_shares/pop_weighted_energy_totals_s{simpl}_{clusters}.csv",
+        district_heat_share="resources/demand/heat/district_heat_share_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv",
+    output:
+        existing_heating_distribution="resources/heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv",
+    threads: 1
+    resources:
+        mem_mb=2000,
+    log:
+        RDIR
+        + "/logs/build_existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.log",
+    benchmark:
+        RDIR
+        +"/benchmarks/build_existing_heating_distribution/{demand}_s{simpl}_{clusters}_{planning_horizons}"
+    script:
+        "scripts/build_existing_heating_distribution.py"
+
+
+if config["foresight"] == "myopic":
+
+    rule add_existing_baseyear:
+        params:
+            baseyear=config["scenario"]["planning_horizons"][0],
+            sector=config["sector"],
+            existing_capacities=config["existing_capacities"],
+            costs=config["costs"],
+        input:
+            network=RDIR
+            + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+            powerplants=pypsaearth("resources/powerplants.csv"),
+            busmap_s=pypsaearth("resources/bus_regions/busmap_elec_s{simpl}.csv"),
+            busmap=pypsaearth(
+                "resources/bus_regions/busmap_elec_s{simpl}_{clusters}.csv"
+            ),
+            clustered_pop_layout="resources/population_shares/pop_layout_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
+            costs=CDIR
+            + "costs_{}.csv".format(config["scenario"]["planning_horizons"][0]),
+            cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc",
+            cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc",
+            existing_heating_distribution="resources/heating/existing_heating_distribution_{demand}_s{simpl}_{clusters}_{planning_horizons}.csv",
+        output:
+            RDIR
+            + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+        wildcard_constraints:
+            # TODO: The first planning_horizon needs to be aligned across scenarios
+            # snakemake does not support passing functions to wildcard_constraints
+            # reference: https://github.com/snakemake/snakemake/issues/2703
+            planning_horizons=config["scenario"]["planning_horizons"][0],  #only applies to baseyear
+        threads: 1
+        resources:
+            mem_mb=2000,
+        log:
+            RDIR
+            + "/logs/add_existing_baseyear_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.log",
+        benchmark:
+            RDIR
+            +"/benchmarks/add_existing_baseyear/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export"
+        script:
+            "scripts/add_existing_baseyear.py"
+
+    def input_profile_tech_brownfield(w):
+        return {
+            f"profile_{tech}": pypsaearth(
+                f"resources/renewable_profiles/profile_{tech}.nc"
+            )
+            for tech in config["electricity"]["renewable_carriers"]
+            if tech != "hydro"
+        }
+
+    def solved_previous_horizon(w):
+        planning_horizons = config["scenario"]["planning_horizons"]
+        i = planning_horizons.index(int(w.planning_horizons))
+        planning_horizon_p = str(planning_horizons[i - 1])
+
+        return (
+            RDIR
+            + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_"
+            + planning_horizon_p
+            + "_{discountrate}_{demand}_{h2export}export.nc"
+        )
+
+    rule add_brownfield:
+        params:
+            H2_retrofit=config["sector"]["hydrogen"],
+            H2_retrofit_capacity_per_CH4=config["sector"]["hydrogen"][
+                "H2_retrofit_capacity_per_CH4"
+            ],
+            threshold_capacity=config["existing_capacities"]["threshold_capacity"],
+            snapshots=config["snapshots"],
+            # drop_leap_day=config["enable"]["drop_leap_day"],
+            carriers=config["electricity"]["renewable_carriers"],
+        input:
+            unpack(input_profile_tech_brownfield),
+            simplify_busmap=pypsaearth("resources/bus_regions/busmap_elec_s{simpl}.csv"),
+            cluster_busmap=pypsaearth(
+                "resources/bus_regions/busmap_elec_s{simpl}_{clusters}.csv"
+            ),
+            network=RDIR
+            + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+            network_p=solved_previous_horizon,  #solved network at previous time step
+            costs=CDIR + "costs_{planning_horizons}.csv",
+            cop_soil_total="resources/cops/cop_soil_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc",
+            cop_air_total="resources/cops/cop_air_total_elec_s{simpl}_{clusters}_{planning_horizons}.nc",
+        output:
+            RDIR
+            + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+        threads: 4
+        resources:
+            mem_mb=10000,
+        log:
+            RDIR
+            + "/logs/add_brownfield_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.log",
+        benchmark:
+            (
+                RDIR
+                + "/benchmarks/add_brownfield/elec_s{simpl}_ec_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export"
+            )
+        script:
+            "./scripts/add_brownfield.py"
+
+    ruleorder: add_existing_baseyear > add_brownfield
+
+    rule solve_network_myopic:
+        params:
+            solving=config["solving"],
+            foresight=config["foresight"],
+            planning_horizons=config["scenario"]["planning_horizons"],
+            co2_sequestration_potential=config["scenario"].get(
+                "co2_sequestration_potential", 200
+            ),
+        input:
+            overrides="data/override_component_attrs",
+            network=RDIR
+            + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+            costs=CDIR + "costs_{planning_horizons}.csv",
+            configs=SDIR + "/configs/config.yaml",  # included to trigger copy_config rule
+        output:
+            network=RDIR
+            + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+            # config=RDIR
+            # + "/configs/config.elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.yaml",
+        shadow:
+            "shallow"
+        log:
+            solver=RDIR
+            + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_solver.log",
+            python=RDIR
+            + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_python.log",
+            memory=RDIR
+            + "/logs/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export_memory.log",
+        threads: 25
+        resources:
+            mem_mb=config["solving"]["mem"],
+        benchmark:
+            (
+                RDIR
+                + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export"
+            )
+        script:
+            "./scripts/solve_network.py"
+
+    rule solve_all_networks_myopic:
+        input:
+            networks=expand(
+                RDIR
+                + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
+                **config["scenario"],
+                **config["costs"],
+                **config["export"],
+            ),

config.default.yaml

@@ -54,6 +54,7 @@ demand_data:
 
 enable:
   retrieve_cost_data: true # if true, the workflow overwrites the cost data saved in data/costs again
+  retrieve_irena: true  #If true, downloads the IRENA data
 
 fossil_reserves:
   oil: 100 #TWh Maybe reduntant
@@ -119,6 +120,17 @@ solar_thermal:
     slope: 45.
     azimuth: 180.
 
+existing_capacities:
+  grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030]
+  grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020
+  threshold_capacity: 10
+  default_heating_lifetime: 20
+  conventional_carriers:
+  - lignite
+  - coal
+  - oil
+  - uranium
+
 sector:
   gas:
     spatial_gas: true # ALWAYS TRUE
@@ -127,6 +139,7 @@ sector:
     network_data_GGIT_status: ['Construction', 'Operating', 'Idle', 'Shelved', 'Mothballed', 'Proposed']
   hydrogen:
     network: true
+    H2_retrofit_capacity_per_CH4: 0.6
     network_limit: 2000 #GWkm
     network_routes: gas # "gas or "greenfield". If "gas"  ->  the network data are fetched from ["sector"]["gas"]["network_data"]. If "greenfield"  -> the network follows the topology of electrical transmission lines
     gas_network_repurposing: true # If true -> ["sector"]["gas"]["network"] is automatically false

data/existing_infrastructure/existing_heating_raw.csv

@@ -0,0 +1,32 @@
+,gas boiler,coal boiler,oil boiler,resistive heater,air heat pump,ground heat pump
+Austria,9.32,0.4,15.42,0,0.72,1.077
+Belgium,28.39,1.19,19.53,3.14,0.17,0.061
+Bulgaria,0.16,3.68,0.04,3.46,1.01,0.045
+Croatia,8.39,0.03,2.88,1.53,0,0
+Czech Republic,9.26,1.02,0.1,2.73,0.35,0.263
+Denmark,4.82,0,3.67,2.19,1.9,0.381
+Estonia,0.22,0.02,0.12,0.27,0.33,0.1
+Finland,0,0.04,3.79,10.3,1.98,0.58
+France,76.85,1.03,46.03,87.24,26.14,1.97
+Germany,131.09,0.44,132.04,0,2.38,3.29
+Greece,2.17,0.03,18.13,5.91,0,0
+Hungary,21.21,1.3,0.04,0.06,0.03,0.035
+Ireland,4.32,0.8,4.85,1.03,0.03,0.03
+Italy,112.68,1.89,3.33,6.61,54.98,0.6
+Latvia,1.53,0.4,0,0.03,0,0
+Lithuania,0,0,0,0,0.01,0.02
+Luxembourg,0.79,0,0.77,0.09,0.01,0.001
+Netherlands,81.41,0,0.1,0.1,1.82,0.849
+Poland,8.25,24.75,9.04,5.96,0.01,0.04
+Portugal,4.79,0,0.2,21.26,1.58,0.064
+Romania,16.56,0.32,0.03,0.72,0,0
+Slovakia,8.05,0.19,0.01,0.55,0.06,0.015
+Slovenia,0.4,0,1.08,0.4,0.03,0.056
+Spain,48.99,0.51,17.95,56.58,1.15,0.016
+Sweden,1.01,0,0.77,3.76,3.42,4.813
+United Kingdom,160.49,1.26,7.39,13.81,0.81,0.21
+Norway,,,,,2.91,0.334
+Switzerland,,,,,1,0.849
+Serbia,,,,,,
+Bosnia Herzegovina,,,,,,
+DEFAULT,,,,,,