From 905dbbf9f18c03e852bd7e3e3093f70bcb354258 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 16 Aug 2023 16:18:58 +0200
Subject: [PATCH 01/58] adding a working script

---
 scripts/build_energy_totals.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 scripts/build_energy_totals.py

diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py
new file mode 100644
index 00000000..e69de29b

From 0ec3c8e20b13c8d99d0ba36814843bcacc2e62a9 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 16 Aug 2023 17:14:37 +0200
Subject: [PATCH 02/58] add the rule in the snakefile

---
 Snakefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Snakefile b/Snakefile
index 3ea23ebd..8a180557 100644
--- a/Snakefile
+++ b/Snakefile
@@ -233,6 +233,12 @@ rule prepare_heat_data:
     script:
         "scripts/prepare_heat_data.py"
 
+rule build_energy_totals:
+    output:
+        energy_totals_base="data/energy_totals_base.csv"
+        energy_totals="data/energy_totals_{demand}_{planning_horizon}.csv"
+    script:
+        "scripts/build_energy_totals.py"
 
 rule build_solar_thermal_profiles:
     input:

From 9dbf015de68bb3039cc2e74f0c24f5f626f51d2e Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 16 Aug 2023 18:01:42 +0200
Subject: [PATCH 03/58] rename script to add new

---
 scripts/build_base_energy_totals.py | 176 ++++++++++++++++++++++++++++
 scripts/build_energy_totals.py      |   0
 2 files changed, 176 insertions(+)
 create mode 100644 scripts/build_base_energy_totals.py
 delete mode 100644 scripts/build_energy_totals.py

diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
new file mode 100644
index 00000000..9fcb85af
--- /dev/null
+++ b/scripts/build_base_energy_totals.py
@@ -0,0 +1,176 @@
+import os
+import requests
+import py7zr
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import country_converter as coco
+from pathlib import Path
+from io import BytesIO
+from urllib.request import urlopen
+from zipfile import ZipFile
+import sys
+from helpers import sets_path_to_root, three_2_two_digits_country
+
+pd.options.mode.chained_assignment = None 
+
+
+def calc_sector(sector):
+    for country in countries:
+        print(country, sector)
+        df_co = df_yr[df_yr.country == country]
+
+        if sector!= "navigation":
+            df_sector = df_co.loc[df["Commodity - Transaction"].str.lower().str.contains(sector)]    
+            #assert df_yr[df_yr["Commodity - Transaction"].str.contains(sector)]["Unit"].unique() == 'Metric tons,  thousand', "Not all quantities have the expected unit: {}".format(expected_unit)
+        else:
+            df_sector = df_co.loc[(df["Commodity - Transaction"].str.lower().str.contains(sector))|(df["Commodity - Transaction"].str.lower().str.contains("marine bunkers"))]    
+
+        if df_sector.empty:
+            pass
+        else:
+            index_mass = df_sector.loc[df_sector["Unit"]== 'Metric tons,  thousand'].index
+            df_sector.loc[index_mass, "Quantity_TWh"] = df_sector.loc[index_mass].apply(lambda x: x["Quantity"] * fuels_conv_toTWh[x["Commodity"]], axis=1)
+
+            index_energy = df_sector[df_sector["Unit"]== 'Kilowatt-hours, million'].index
+            df_sector.loc[index_energy, "Quantity_TWh"] = df_sector.loc[index_energy].apply(lambda x: x["Quantity"] / 1e3, axis=1)
+            
+            index_energy_TJ = df_sector[df_sector["Unit"]== 'Terajoules'].index
+            df_sector.loc[index_energy_TJ, "Quantity_TWh"] = df_sector.loc[index_energy_TJ].apply(lambda x: x["Quantity"] / 3600, axis=1)
+            
+
+            index_volume = df_sector[df_sector["Unit"]== 'Cubic metres, thousand'].index
+            df_sector.loc[index_volume, "Quantity_TWh"] = df_sector.loc[index_volume].apply(lambda x: x["Quantity"] * fuels_conv_toTWh[x["Commodity"]], axis=1)
+            
+            sectors_dfs[sector]=df_sector.copy() 
+
+            if sector ==  "consumption by households":
+                energy_totals_base.at[country, "electricity residential"] = df_sector[(df_sector.Commodity=="Electricity") | df_sector.Commodity.isin(other_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "residential oil"] = df_sector[df_sector.Commodity.isin(oil_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "residential biomass"] = df_sector[df_sector.Commodity.isin(biomass_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "residential gas"] = df_sector[df_sector.Commodity.isin(gas_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "total residential space"] = df_sector[df_sector.Commodity.isin(heat)].Quantity_TWh.sum().round(4)*0.6
+                energy_totals_base.at[country, "total residential water"] = df_sector[df_sector.Commodity.isin(heat)].Quantity_TWh.sum().round(4)*0.4
+            
+            elif sector== "services":            
+                energy_totals_base.at[country, "services electricity"] = df_sector[(df_sector.Commodity=="Electricity") | df_sector.Commodity.isin(other_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "services oil"] = df_sector[df_sector.Commodity.isin(oil_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "services biomass"] = df_sector[df_sector.Commodity.isin(biomass_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "services gas"] = df_sector[df_sector.Commodity.isin(gas_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "total services space"] = df_sector[df_sector.Commodity.isin(heat)].Quantity_TWh.sum().round(4)*0.6
+                energy_totals_base.at[country, "total services water"] = df_sector[df_sector.Commodity.isin(heat)].Quantity_TWh.sum().round(4)*0.4
+
+            elif sector == "road":
+                energy_totals_base.at[country, "total road"] = df_sector.Quantity_TWh.sum().round(4)
+
+            elif sector == "agriculture":   
+                energy_totals_base.at[country, "agriculture electricity"] = df_sector[(df_sector.Commodity=="Electricity") | df_sector.Commodity.isin(other_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "agriculture oil"] = df_sector[df_sector.Commodity.isin(oil_fuels)].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "agriculture biomass"] = df_sector[df_sector.Commodity.isin(biomass_fuels)].Quantity_TWh.sum().round(4)
+                #energy_totals_base.at[country, "electricity rail"] = df_house[(df_house.Commodity=="Electricity")].Quantity_TWh.sum().round(4)
+
+            elif sector == "rail":
+                energy_totals_base.at[country, "total rail"] = df_sector[(df_sector.Commodity=="Gas Oil/ Diesel Oil")|(df_sector.Commodity=="Biodiesel")].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "electricity rail"] = df_sector[(df_sector.Commodity=="Electricity")].Quantity_TWh.sum().round(4)
+
+            elif sector == "aviation":
+                energy_totals_base.at[country, "total international aviation"] = df_sector[(df_sector.Commodity == "Kerosene-type Jet Fuel") & (df_sector.Transaction == "International aviation bunkers")].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "total domestic aviation"] = df_sector[(df_sector.Commodity == "Kerosene-type Jet Fuel") & (df_sector.Transaction == "Consumption by domestic aviation")].Quantity_TWh.sum().round(4)
+
+            elif sector == "navigation":
+                energy_totals_base.at[country, "total international navigation"] = df_sector[df_sector.Transaction=="International marine bunkers"].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "total domestic navigation"] = df_sector[df_sector.Transaction=="Consumption by domestic navigation"].Quantity_TWh.sum().round(4)
+            
+            else:
+                print("wrong sector")
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from helpers import mock_snakemake, sets_path_to_root
+
+        os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+        snakemake = mock_snakemake(
+            "build_energy_totals",
+            simpl="",
+            clusters=10,
+            demand="DF",
+            planning_horizons=2030,
+        )
+        sets_path_to_root("pypsa-earth-sec")
+
+    energy_stat_database = pd.read_excel(snakemake.input.unsd_paths, index_col=0, header=0) #pd.read_excel("/nfs/home/haz43975/pypsa-earth-sec/scripts/Energy_Statistics_Database.xlsx"
+
+    # Load the links and make a dictionary
+    df = energy_stat_database.copy()
+    df = df.dropna(axis=0, subset=['Link'])
+    df = df.to_dict('dict')
+    d = df['Link'] 
+
+    # Feed the dictionary of links to the for loop, download and unzip all files
+    for key, value in d.items():
+        zipurl = value
+
+        with urlopen(zipurl) as zipresp:
+            with ZipFile(BytesIO(zipresp.read())) as zfile:
+                zfile.extractall("/nfs/home/haz43975/pypsa-earth-sec/data/demand/unsd")
+
+                path = "/nfs/home/haz43975/pypsa-earth-sec/data/demand/unsd"
+
+    # Get the files from the path provided in the OP
+    all_files = Path(path).glob('*.txt')
+
+    # Create a dataframe from all downloaded files
+    df = pd.concat((pd.read_csv(f, encoding='utf8', sep=';') for f in all_files), ignore_index=True)
+
+    # Split 'Commodity', 'Transaction' column to two
+    df[['Commodity', 'Transaction', 'extra']] = df['Commodity - Transaction'].str.split(' - ', expand=True)
+
+    # Remove Foootnote and Estimate from 'Commodity - Transaction' column
+    #df = df.loc[df['Commodity - Transaction'] != 'Footnote']
+    #df = df.loc[df['Commodity - Transaction'] != 'Estimate']
+
+    # Create a column with iso2 country code
+    cc = coco.CountryConverter()
+    Country = pd.Series(df['Country or Area'])
+
+    df["country"] = cc.pandas_convert(series=Country, to='ISO2', not_found='not found')  
+
+    # remove countries or areas that have no iso2 such as former countries names
+    df = df.loc[df['country'] != 'not found']
+
+    # Convert country column that contains lists for some country names that are identified with more than one country.
+    df["country"] = df["country"].astype(str)
+
+    # Remove all iso2 conversions for some country names that are identified with more than one country.
+    df = df[~df.country.str.contains(',', na=False)].reset_index(drop=True)
+    
+    fuels_conv_toTWh = {"Gas Oil/ Diesel Oil":0.01194, "Motor Gasoline":0.01230, "Kerosene-type Jet Fuel": 0.01225, "Aviation gasoline":0.01230, "Biodiesel": 0.01022,"Natural gas liquids": 0.01228,
+    "Biogasoline": 0.007444, "Bitumen": 0.01117, "Fuel oil": 0.01122, 'Liquefied petroleum gas (LPG)':0.01313, "Liquified Petroleum Gas (LPG)": 0.01313,"Lubricants":0.01117, "Naphtha": 0.01236, "Fuelwood": 0.00254, "Charcoal":0.00819,
+    "Patent fuel": 0.00575, "Brown coal briquettes":0.00575, "Hard coal":0.007167, "Other bituminous coal":0.005556, "Anthracite":0.005, "Peat":0.00271, "Peat products":0.00271, "Lignite":0.003889,
+    "Brown coal": 0.003889, "Sub-bituminous coal": 0.005555, "Coke-oven coke":0.0002778, "Coke oven coke":0.0002778,"Coke Oven Coke":0.0002778, "Gasoline-type jet fuel":0.01230, "Conventional crude oil": 0.01175,
+    'Brown Coal Briquettes': 0.00575, "Refinery Gas":0.01375, "Petroleum coke":0.009028, "Coking coal": 0.007833, "Peat Products":0.00271, 'Petroleum Coke':0.009028}
+
+    year=2019
+
+    countries = ["BR", "AR", "EG"]
+
+    df_yr = df[df.Year==year]
+    df_yr = df_yr[df_yr.country.isin(countries)]
+    energy_totals_cols = pd.read_csv("/nfs/home/haz43975/pypsa-earth-sec/data/energy_totals_DF_2030.csv").columns
+    energy_totals = pd.DataFrame(columns=energy_totals_cols, index=countries)
+
+    oil_fuels=["Patent fuel","Gas Oil/ Diesel Oil","Motor Gasoline", "Liquefied petroleum gas (LPG)"]
+    gas_fuels=["Natural gas (including LNG)","Gasworks Gas"]
+    biomass_fuels=["Biodiesel", "Biogases", "Fuelwood"]
+    other_fuels = ["Charcoal","Brown coal briquettes","Other bituminous coal"]
+    heat = ["Heat", "Direct use of geothermal heat", "Direct use of solar thermal heat"]
+    energy_totals_cols = pd.read_csv("/nfs/home/haz43975/pypsa-earth-sec/data/energy_totals_DF_2030.csv").columns
+    energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries)
+    sectors_dfs={}
+
+    sectors = [ "consumption by households", "road","rail", "aviation", "navigation","agriculture", "services"]
+
+    for sector in sectors:
+        calc_sector(sector)
+    energy_totals_base.dropna(axis=1, how="all")
\ No newline at end of file
diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py
deleted file mode 100644
index e69de29b..00000000

From 1397b928cb9b51a8b0b9fa8c0f2f9d8205d9aa6c Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 16 Aug 2023 18:32:39 +0200
Subject: [PATCH 04/58] in-script documentation and option to trigger download

---
 scripts/build_base_energy_totals.py | 54 +++++++++++++++++++----------
 1 file changed, 35 insertions(+), 19 deletions(-)

diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index 9fcb85af..5bc5367f 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -1,4 +1,5 @@
 import os
+import glob
 import requests
 import py7zr
 import numpy as np
@@ -84,6 +85,7 @@ def calc_sector(sector):
             else:
                 print("wrong sector")
 
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from helpers import mock_snakemake, sets_path_to_root
@@ -91,7 +93,7 @@ def calc_sector(sector):
         os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
         snakemake = mock_snakemake(
-            "build_energy_totals",
+            "build_base_energy_totals",
             simpl="",
             clusters=10,
             demand="DF",
@@ -107,18 +109,25 @@ def calc_sector(sector):
     df = df.to_dict('dict')
     d = df['Link'] 
 
-    # Feed the dictionary of links to the for loop, download and unzip all files
-    for key, value in d.items():
-        zipurl = value
+    if snakemake.config["demand_data"]["update_data"]:
+    # Delete and existing files to avoid duplication and double counting
+            
+        files = glob.glob('data/demand/unsd/*.txt')
+        for f in files:
+            os.remove(f)
+
+        # Feed the dictionary of links to the for loop, download and unzip all files
+        for key, value in d.items():
+            zipurl = value
 
-        with urlopen(zipurl) as zipresp:
-            with ZipFile(BytesIO(zipresp.read())) as zfile:
-                zfile.extractall("/nfs/home/haz43975/pypsa-earth-sec/data/demand/unsd")
+            with urlopen(zipurl) as zipresp:
+                with ZipFile(BytesIO(zipresp.read())) as zfile:
+                    zfile.extractall("data/demand/unsd")
 
-                path = "/nfs/home/haz43975/pypsa-earth-sec/data/demand/unsd"
+                    path = "data/demand/unsd"
 
     # Get the files from the path provided in the OP
-    all_files = Path(path).glob('*.txt')
+    all_files = Path("data/demand/unsd").glob('*.txt')
 
     # Create a dataframe from all downloaded files
     df = pd.concat((pd.read_csv(f, encoding='utf8', sep=';') for f in all_files), ignore_index=True)
@@ -127,8 +136,8 @@ def calc_sector(sector):
     df[['Commodity', 'Transaction', 'extra']] = df['Commodity - Transaction'].str.split(' - ', expand=True)
 
     # Remove Foootnote and Estimate from 'Commodity - Transaction' column
-    #df = df.loc[df['Commodity - Transaction'] != 'Footnote']
-    #df = df.loc[df['Commodity - Transaction'] != 'Estimate']
+    df = df.loc[df['Commodity - Transaction'] != 'Footnote']
+    df = df.loc[df['Commodity - Transaction'] != 'Estimate']
 
     # Create a column with iso2 country code
     cc = coco.CountryConverter()
@@ -145,32 +154,39 @@ def calc_sector(sector):
     # Remove all iso2 conversions for some country names that are identified with more than one country.
     df = df[~df.country.str.contains(',', na=False)].reset_index(drop=True)
     
+    # Create a dictionary with all the conversion factors from ktons or m3 to TWh based on https://unstats.un.org/unsd/energy/balance/2014/05.pdf
     fuels_conv_toTWh = {"Gas Oil/ Diesel Oil":0.01194, "Motor Gasoline":0.01230, "Kerosene-type Jet Fuel": 0.01225, "Aviation gasoline":0.01230, "Biodiesel": 0.01022,"Natural gas liquids": 0.01228,
     "Biogasoline": 0.007444, "Bitumen": 0.01117, "Fuel oil": 0.01122, 'Liquefied petroleum gas (LPG)':0.01313, "Liquified Petroleum Gas (LPG)": 0.01313,"Lubricants":0.01117, "Naphtha": 0.01236, "Fuelwood": 0.00254, "Charcoal":0.00819,
     "Patent fuel": 0.00575, "Brown coal briquettes":0.00575, "Hard coal":0.007167, "Other bituminous coal":0.005556, "Anthracite":0.005, "Peat":0.00271, "Peat products":0.00271, "Lignite":0.003889,
     "Brown coal": 0.003889, "Sub-bituminous coal": 0.005555, "Coke-oven coke":0.0002778, "Coke oven coke":0.0002778,"Coke Oven Coke":0.0002778, "Gasoline-type jet fuel":0.01230, "Conventional crude oil": 0.01175,
     'Brown Coal Briquettes': 0.00575, "Refinery Gas":0.01375, "Petroleum coke":0.009028, "Coking coal": 0.007833, "Peat Products":0.00271, 'Petroleum Coke':0.009028}
 
-    year=2019
-
-    countries = ["BR", "AR", "EG"]
+    #Fetch country list and demand base year from the config file
+    year=snakemake.config["demand_data"]["base_year"]
+    countries = snakemake.config["countries"]
 
+    # Filter for the year and country
     df_yr = df[df.Year==year]
     df_yr = df_yr[df_yr.country.isin(countries)]
+
+    # Create an empty dataframe for energy_totals_base
     energy_totals_cols = pd.read_csv("/nfs/home/haz43975/pypsa-earth-sec/data/energy_totals_DF_2030.csv").columns
-    energy_totals = pd.DataFrame(columns=energy_totals_cols, index=countries)
+    energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries)
 
+    # Lists that combine the different fuels in the dataset to the model's carriers
     oil_fuels=["Patent fuel","Gas Oil/ Diesel Oil","Motor Gasoline", "Liquefied petroleum gas (LPG)"]
     gas_fuels=["Natural gas (including LNG)","Gasworks Gas"]
     biomass_fuels=["Biodiesel", "Biogases", "Fuelwood"]
     other_fuels = ["Charcoal","Brown coal briquettes","Other bituminous coal"]
     heat = ["Heat", "Direct use of geothermal heat", "Direct use of solar thermal heat"]
-    energy_totals_cols = pd.read_csv("/nfs/home/haz43975/pypsa-earth-sec/data/energy_totals_DF_2030.csv").columns
-    energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries)
+
+    # Create a dictionary to save the data if need to be checked
     sectors_dfs={}
 
+    # Run the function that processes the data for all the sectors
     sectors = [ "consumption by households", "road","rail", "aviation", "navigation","agriculture", "services"]
-
     for sector in sectors:
         calc_sector(sector)
-    energy_totals_base.dropna(axis=1, how="all")
\ No newline at end of file
+    
+    #Export the base energy totals file
+    energy_totals_base.dropna(axis=1, how="all").to_csv(snakemake.output.energy_totals_base)
\ No newline at end of file

From e6f9d715ee8d59906c3051a2607ca84714406ab1 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 16 Aug 2023 18:33:26 +0200
Subject: [PATCH 05/58] update config files

---
 config.default.yaml    | 4 ++++
 test/config.test1.yaml | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/config.default.yaml b/config.default.yaml
index 0ce95b57..807796de 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -39,6 +39,10 @@ clustering_options:
 
 countries: ['MA']
 
+demand_data:
+  update_data: true
+  base_year: 2019
+  
 H2_network: false
 H2_network_limit: 2000 #GWkm
 
diff --git a/test/config.test1.yaml b/test/config.test1.yaml
index 045345b6..3d01c97d 100644
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@@ -40,6 +40,10 @@ clustering_options:
 
 countries: ['NG', 'BJ']
 
+demand_data:
+  update_data: true
+  base_year: 2019
+
 H2_network: false
 H2_network_limit: 2000 #GWkm
 

From 5c3dfe5bd869ff6fb42be43f73a4ae21fb320996 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 16 Aug 2023 18:33:59 +0200
Subject: [PATCH 06/58] adapt snakefile rule

---
 Snakefile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Snakefile b/Snakefile
index 8a180557..cca10711 100644
--- a/Snakefile
+++ b/Snakefile
@@ -233,10 +233,11 @@ rule prepare_heat_data:
     script:
         "scripts/prepare_heat_data.py"
 
-rule build_energy_totals:
+rule build_base_energy_totals:
+    input:
+        unsd_paths="/nfs/home/haz43975/pypsa-earth-sec/scripts/Energy_Statistics_Database.xlsx"
     output:
-        energy_totals_base="data/energy_totals_base.csv"
-        energy_totals="data/energy_totals_{demand}_{planning_horizon}.csv"
+        energy_totals_base="data/energy_totals_base.csv",
     script:
         "scripts/build_energy_totals.py"
 

From 56c97afe137d168c9dfce9dd17e5a35e314b9af5 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 16 Aug 2023 18:53:45 +0200
Subject: [PATCH 07/58] remove unnecessay commands and files

---
 .gitignore                          | 5 +++++
 scripts/build_base_energy_totals.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index fafa28c2..9fa19053 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,6 +37,9 @@ data/industry_sector_ratios_NZ_2030.csv
 #/data/Industrial_Database.csv
 /data/retro/tabula-calculator-calcsetbuilding.csv
 /data/nuts*
+/data/demand/unsd
+/data/ports.csv
+/data/airports.csv
 /scripts/old
 doc/_build
 /cutouts
@@ -46,6 +49,8 @@ doc/_build
 /scripts/data
 /sample_pypsa_eur
 
+/notebooks
+
 # File extensions
 *.org
 *.nc
diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index 5bc5367f..d65dc94a 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -18,7 +18,7 @@
 
 def calc_sector(sector):
     for country in countries:
-        print(country, sector)
+        #print(country, sector)
         df_co = df_yr[df_yr.country == country]
 
         if sector!= "navigation":

From ef8ccfd597ff935efc6ecf9d75273f0ec81393c6 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 17 Aug 2023 07:59:34 +0000
Subject: [PATCH 08/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 Snakefile                           |   4 +-
 config.default.yaml                 |   2 +-
 scripts/build_base_energy_totals.py | 380 +++++++++++++++++++++-------
 3 files changed, 289 insertions(+), 97 deletions(-)

diff --git a/Snakefile b/Snakefile
index cca10711..87e02c84 100644
--- a/Snakefile
+++ b/Snakefile
@@ -233,14 +233,16 @@ rule prepare_heat_data:
     script:
         "scripts/prepare_heat_data.py"
 
+
 rule build_base_energy_totals:
     input:
-        unsd_paths="/nfs/home/haz43975/pypsa-earth-sec/scripts/Energy_Statistics_Database.xlsx"
+        unsd_paths="/nfs/home/haz43975/pypsa-earth-sec/scripts/Energy_Statistics_Database.xlsx",
     output:
         energy_totals_base="data/energy_totals_base.csv",
     script:
         "scripts/build_energy_totals.py"
 
+
 rule build_solar_thermal_profiles:
     input:
         pop_layout_total="resources/population_shares/pop_layout_total.nc",
diff --git a/config.default.yaml b/config.default.yaml
index 807796de..6eedbfa6 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -42,7 +42,7 @@ countries: ['MA']
 demand_data:
   update_data: true
   base_year: 2019
-  
+
 H2_network: false
 H2_network_limit: 2000 #GWkm
 
diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index d65dc94a..8dcc34cc 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -1,87 +1,221 @@
-import os
+# -*- coding: utf-8 -*-
 import glob
-import requests
-import py7zr
-import numpy as np
-import matplotlib.pyplot as plt
-import pandas as pd
-import country_converter as coco
-from pathlib import Path
+import os
+import sys
 from io import BytesIO
+from pathlib import Path
 from urllib.request import urlopen
 from zipfile import ZipFile
-import sys
+
+import country_converter as coco
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import py7zr
+import requests
 from helpers import sets_path_to_root, three_2_two_digits_country
 
-pd.options.mode.chained_assignment = None 
+pd.options.mode.chained_assignment = None
 
 
 def calc_sector(sector):
     for country in countries:
-        #print(country, sector)
+        # print(country, sector)
         df_co = df_yr[df_yr.country == country]
 
-        if sector!= "navigation":
-            df_sector = df_co.loc[df["Commodity - Transaction"].str.lower().str.contains(sector)]    
-            #assert df_yr[df_yr["Commodity - Transaction"].str.contains(sector)]["Unit"].unique() == 'Metric tons,  thousand', "Not all quantities have the expected unit: {}".format(expected_unit)
+        if sector != "navigation":
+            df_sector = df_co.loc[
+                df["Commodity - Transaction"].str.lower().str.contains(sector)
+            ]
+            # assert df_yr[df_yr["Commodity - Transaction"].str.contains(sector)]["Unit"].unique() == 'Metric tons,  thousand', "Not all quantities have the expected unit: {}".format(expected_unit)
         else:
-            df_sector = df_co.loc[(df["Commodity - Transaction"].str.lower().str.contains(sector))|(df["Commodity - Transaction"].str.lower().str.contains("marine bunkers"))]    
+            df_sector = df_co.loc[
+                (df["Commodity - Transaction"].str.lower().str.contains(sector))
+                | (
+                    df["Commodity - Transaction"]
+                    .str.lower()
+                    .str.contains("marine bunkers")
+                )
+            ]
 
         if df_sector.empty:
             pass
         else:
-            index_mass = df_sector.loc[df_sector["Unit"]== 'Metric tons,  thousand'].index
-            df_sector.loc[index_mass, "Quantity_TWh"] = df_sector.loc[index_mass].apply(lambda x: x["Quantity"] * fuels_conv_toTWh[x["Commodity"]], axis=1)
-
-            index_energy = df_sector[df_sector["Unit"]== 'Kilowatt-hours, million'].index
-            df_sector.loc[index_energy, "Quantity_TWh"] = df_sector.loc[index_energy].apply(lambda x: x["Quantity"] / 1e3, axis=1)
-            
-            index_energy_TJ = df_sector[df_sector["Unit"]== 'Terajoules'].index
-            df_sector.loc[index_energy_TJ, "Quantity_TWh"] = df_sector.loc[index_energy_TJ].apply(lambda x: x["Quantity"] / 3600, axis=1)
-            
-
-            index_volume = df_sector[df_sector["Unit"]== 'Cubic metres, thousand'].index
-            df_sector.loc[index_volume, "Quantity_TWh"] = df_sector.loc[index_volume].apply(lambda x: x["Quantity"] * fuels_conv_toTWh[x["Commodity"]], axis=1)
-            
-            sectors_dfs[sector]=df_sector.copy() 
-
-            if sector ==  "consumption by households":
-                energy_totals_base.at[country, "electricity residential"] = df_sector[(df_sector.Commodity=="Electricity") | df_sector.Commodity.isin(other_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "residential oil"] = df_sector[df_sector.Commodity.isin(oil_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "residential biomass"] = df_sector[df_sector.Commodity.isin(biomass_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "residential gas"] = df_sector[df_sector.Commodity.isin(gas_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "total residential space"] = df_sector[df_sector.Commodity.isin(heat)].Quantity_TWh.sum().round(4)*0.6
-                energy_totals_base.at[country, "total residential water"] = df_sector[df_sector.Commodity.isin(heat)].Quantity_TWh.sum().round(4)*0.4
-            
-            elif sector== "services":            
-                energy_totals_base.at[country, "services electricity"] = df_sector[(df_sector.Commodity=="Electricity") | df_sector.Commodity.isin(other_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "services oil"] = df_sector[df_sector.Commodity.isin(oil_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "services biomass"] = df_sector[df_sector.Commodity.isin(biomass_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "services gas"] = df_sector[df_sector.Commodity.isin(gas_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "total services space"] = df_sector[df_sector.Commodity.isin(heat)].Quantity_TWh.sum().round(4)*0.6
-                energy_totals_base.at[country, "total services water"] = df_sector[df_sector.Commodity.isin(heat)].Quantity_TWh.sum().round(4)*0.4
+            index_mass = df_sector.loc[
+                df_sector["Unit"] == "Metric tons,  thousand"
+            ].index
+            df_sector.loc[index_mass, "Quantity_TWh"] = df_sector.loc[index_mass].apply(
+                lambda x: x["Quantity"] * fuels_conv_toTWh[x["Commodity"]], axis=1
+            )
+
+            index_energy = df_sector[
+                df_sector["Unit"] == "Kilowatt-hours, million"
+            ].index
+            df_sector.loc[index_energy, "Quantity_TWh"] = df_sector.loc[
+                index_energy
+            ].apply(lambda x: x["Quantity"] / 1e3, axis=1)
+
+            index_energy_TJ = df_sector[df_sector["Unit"] == "Terajoules"].index
+            df_sector.loc[index_energy_TJ, "Quantity_TWh"] = df_sector.loc[
+                index_energy_TJ
+            ].apply(lambda x: x["Quantity"] / 3600, axis=1)
+
+            index_volume = df_sector[
+                df_sector["Unit"] == "Cubic metres, thousand"
+            ].index
+            df_sector.loc[index_volume, "Quantity_TWh"] = df_sector.loc[
+                index_volume
+            ].apply(lambda x: x["Quantity"] * fuels_conv_toTWh[x["Commodity"]], axis=1)
+
+            sectors_dfs[sector] = df_sector.copy()
+
+            if sector == "consumption by households":
+                energy_totals_base.at[country, "electricity residential"] = (
+                    df_sector[
+                        (df_sector.Commodity == "Electricity")
+                        | df_sector.Commodity.isin(other_fuels)
+                    ]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "residential oil"] = (
+                    df_sector[df_sector.Commodity.isin(oil_fuels)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "residential biomass"] = (
+                    df_sector[df_sector.Commodity.isin(biomass_fuels)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "residential gas"] = (
+                    df_sector[df_sector.Commodity.isin(gas_fuels)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "total residential space"] = (
+                    df_sector[df_sector.Commodity.isin(heat)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                    * 0.6
+                )
+                energy_totals_base.at[country, "total residential water"] = (
+                    df_sector[df_sector.Commodity.isin(heat)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                    * 0.4
+                )
+
+            elif sector == "services":
+                energy_totals_base.at[country, "services electricity"] = (
+                    df_sector[
+                        (df_sector.Commodity == "Electricity")
+                        | df_sector.Commodity.isin(other_fuels)
+                    ]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "services oil"] = (
+                    df_sector[df_sector.Commodity.isin(oil_fuels)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "services biomass"] = (
+                    df_sector[df_sector.Commodity.isin(biomass_fuels)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "services gas"] = (
+                    df_sector[df_sector.Commodity.isin(gas_fuels)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "total services space"] = (
+                    df_sector[df_sector.Commodity.isin(heat)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                    * 0.6
+                )
+                energy_totals_base.at[country, "total services water"] = (
+                    df_sector[df_sector.Commodity.isin(heat)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                    * 0.4
+                )
 
             elif sector == "road":
-                energy_totals_base.at[country, "total road"] = df_sector.Quantity_TWh.sum().round(4)
-
-            elif sector == "agriculture":   
-                energy_totals_base.at[country, "agriculture electricity"] = df_sector[(df_sector.Commodity=="Electricity") | df_sector.Commodity.isin(other_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "agriculture oil"] = df_sector[df_sector.Commodity.isin(oil_fuels)].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "agriculture biomass"] = df_sector[df_sector.Commodity.isin(biomass_fuels)].Quantity_TWh.sum().round(4)
-                #energy_totals_base.at[country, "electricity rail"] = df_house[(df_house.Commodity=="Electricity")].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[
+                    country, "total road"
+                ] = df_sector.Quantity_TWh.sum().round(4)
+
+            elif sector == "agriculture":
+                energy_totals_base.at[country, "agriculture electricity"] = (
+                    df_sector[
+                        (df_sector.Commodity == "Electricity")
+                        | df_sector.Commodity.isin(other_fuels)
+                    ]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "agriculture oil"] = (
+                    df_sector[df_sector.Commodity.isin(oil_fuels)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "agriculture biomass"] = (
+                    df_sector[df_sector.Commodity.isin(biomass_fuels)]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                # energy_totals_base.at[country, "electricity rail"] = df_house[(df_house.Commodity=="Electricity")].Quantity_TWh.sum().round(4)
 
             elif sector == "rail":
-                energy_totals_base.at[country, "total rail"] = df_sector[(df_sector.Commodity=="Gas Oil/ Diesel Oil")|(df_sector.Commodity=="Biodiesel")].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "electricity rail"] = df_sector[(df_sector.Commodity=="Electricity")].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "total rail"] = (
+                    df_sector[
+                        (df_sector.Commodity == "Gas Oil/ Diesel Oil")
+                        | (df_sector.Commodity == "Biodiesel")
+                    ]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "electricity rail"] = (
+                    df_sector[(df_sector.Commodity == "Electricity")]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
 
             elif sector == "aviation":
-                energy_totals_base.at[country, "total international aviation"] = df_sector[(df_sector.Commodity == "Kerosene-type Jet Fuel") & (df_sector.Transaction == "International aviation bunkers")].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "total domestic aviation"] = df_sector[(df_sector.Commodity == "Kerosene-type Jet Fuel") & (df_sector.Transaction == "Consumption by domestic aviation")].Quantity_TWh.sum().round(4)
+                energy_totals_base.at[country, "total international aviation"] = (
+                    df_sector[
+                        (df_sector.Commodity == "Kerosene-type Jet Fuel")
+                        & (df_sector.Transaction == "International aviation bunkers")
+                    ]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "total domestic aviation"] = (
+                    df_sector[
+                        (df_sector.Commodity == "Kerosene-type Jet Fuel")
+                        & (df_sector.Transaction == "Consumption by domestic aviation")
+                    ]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
 
             elif sector == "navigation":
-                energy_totals_base.at[country, "total international navigation"] = df_sector[df_sector.Transaction=="International marine bunkers"].Quantity_TWh.sum().round(4)
-                energy_totals_base.at[country, "total domestic navigation"] = df_sector[df_sector.Transaction=="Consumption by domestic navigation"].Quantity_TWh.sum().round(4)
-            
+                energy_totals_base.at[country, "total international navigation"] = (
+                    df_sector[df_sector.Transaction == "International marine bunkers"]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+                energy_totals_base.at[country, "total domestic navigation"] = (
+                    df_sector[
+                        df_sector.Transaction == "Consumption by domestic navigation"
+                    ]
+                    .Quantity_TWh.sum()
+                    .round(4)
+                )
+
             else:
                 print("wrong sector")
 
@@ -101,18 +235,20 @@ def calc_sector(sector):
         )
         sets_path_to_root("pypsa-earth-sec")
 
-    energy_stat_database = pd.read_excel(snakemake.input.unsd_paths, index_col=0, header=0) #pd.read_excel("/nfs/home/haz43975/pypsa-earth-sec/scripts/Energy_Statistics_Database.xlsx"
+    energy_stat_database = pd.read_excel(
+        snakemake.input.unsd_paths, index_col=0, header=0
+    )  # pd.read_excel("/nfs/home/haz43975/pypsa-earth-sec/scripts/Energy_Statistics_Database.xlsx"
 
     # Load the links and make a dictionary
     df = energy_stat_database.copy()
-    df = df.dropna(axis=0, subset=['Link'])
-    df = df.to_dict('dict')
-    d = df['Link'] 
+    df = df.dropna(axis=0, subset=["Link"])
+    df = df.to_dict("dict")
+    d = df["Link"]
 
     if snakemake.config["demand_data"]["update_data"]:
-    # Delete and existing files to avoid duplication and double counting
-            
-        files = glob.glob('data/demand/unsd/*.txt')
+        # Delete and existing files to avoid duplication and double counting
+
+        files = glob.glob("data/demand/unsd/*.txt")
         for f in files:
             os.remove(f)
 
@@ -127,66 +263,120 @@ def calc_sector(sector):
                     path = "data/demand/unsd"
 
     # Get the files from the path provided in the OP
-    all_files = Path("data/demand/unsd").glob('*.txt')
+    all_files = Path("data/demand/unsd").glob("*.txt")
 
     # Create a dataframe from all downloaded files
-    df = pd.concat((pd.read_csv(f, encoding='utf8', sep=';') for f in all_files), ignore_index=True)
+    df = pd.concat(
+        (pd.read_csv(f, encoding="utf8", sep=";") for f in all_files), ignore_index=True
+    )
 
     # Split 'Commodity', 'Transaction' column to two
-    df[['Commodity', 'Transaction', 'extra']] = df['Commodity - Transaction'].str.split(' - ', expand=True)
+    df[["Commodity", "Transaction", "extra"]] = df["Commodity - Transaction"].str.split(
+        " - ", expand=True
+    )
 
     # Remove Foootnote and Estimate from 'Commodity - Transaction' column
-    df = df.loc[df['Commodity - Transaction'] != 'Footnote']
-    df = df.loc[df['Commodity - Transaction'] != 'Estimate']
+    df = df.loc[df["Commodity - Transaction"] != "Footnote"]
+    df = df.loc[df["Commodity - Transaction"] != "Estimate"]
 
     # Create a column with iso2 country code
     cc = coco.CountryConverter()
-    Country = pd.Series(df['Country or Area'])
+    Country = pd.Series(df["Country or Area"])
 
-    df["country"] = cc.pandas_convert(series=Country, to='ISO2', not_found='not found')  
+    df["country"] = cc.pandas_convert(series=Country, to="ISO2", not_found="not found")
 
     # remove countries or areas that have no iso2 such as former countries names
-    df = df.loc[df['country'] != 'not found']
+    df = df.loc[df["country"] != "not found"]
 
     # Convert country column that contains lists for some country names that are identified with more than one country.
     df["country"] = df["country"].astype(str)
 
     # Remove all iso2 conversions for some country names that are identified with more than one country.
-    df = df[~df.country.str.contains(',', na=False)].reset_index(drop=True)
-    
+    df = df[~df.country.str.contains(",", na=False)].reset_index(drop=True)
+
     # Create a dictionary with all the conversion factors from ktons or m3 to TWh based on https://unstats.un.org/unsd/energy/balance/2014/05.pdf
-    fuels_conv_toTWh = {"Gas Oil/ Diesel Oil":0.01194, "Motor Gasoline":0.01230, "Kerosene-type Jet Fuel": 0.01225, "Aviation gasoline":0.01230, "Biodiesel": 0.01022,"Natural gas liquids": 0.01228,
-    "Biogasoline": 0.007444, "Bitumen": 0.01117, "Fuel oil": 0.01122, 'Liquefied petroleum gas (LPG)':0.01313, "Liquified Petroleum Gas (LPG)": 0.01313,"Lubricants":0.01117, "Naphtha": 0.01236, "Fuelwood": 0.00254, "Charcoal":0.00819,
-    "Patent fuel": 0.00575, "Brown coal briquettes":0.00575, "Hard coal":0.007167, "Other bituminous coal":0.005556, "Anthracite":0.005, "Peat":0.00271, "Peat products":0.00271, "Lignite":0.003889,
-    "Brown coal": 0.003889, "Sub-bituminous coal": 0.005555, "Coke-oven coke":0.0002778, "Coke oven coke":0.0002778,"Coke Oven Coke":0.0002778, "Gasoline-type jet fuel":0.01230, "Conventional crude oil": 0.01175,
-    'Brown Coal Briquettes': 0.00575, "Refinery Gas":0.01375, "Petroleum coke":0.009028, "Coking coal": 0.007833, "Peat Products":0.00271, 'Petroleum Coke':0.009028}
-
-    #Fetch country list and demand base year from the config file
-    year=snakemake.config["demand_data"]["base_year"]
+    fuels_conv_toTWh = {
+        "Gas Oil/ Diesel Oil": 0.01194,
+        "Motor Gasoline": 0.01230,
+        "Kerosene-type Jet Fuel": 0.01225,
+        "Aviation gasoline": 0.01230,
+        "Biodiesel": 0.01022,
+        "Natural gas liquids": 0.01228,
+        "Biogasoline": 0.007444,
+        "Bitumen": 0.01117,
+        "Fuel oil": 0.01122,
+        "Liquefied petroleum gas (LPG)": 0.01313,
+        "Liquified Petroleum Gas (LPG)": 0.01313,
+        "Lubricants": 0.01117,
+        "Naphtha": 0.01236,
+        "Fuelwood": 0.00254,
+        "Charcoal": 0.00819,
+        "Patent fuel": 0.00575,
+        "Brown coal briquettes": 0.00575,
+        "Hard coal": 0.007167,
+        "Other bituminous coal": 0.005556,
+        "Anthracite": 0.005,
+        "Peat": 0.00271,
+        "Peat products": 0.00271,
+        "Lignite": 0.003889,
+        "Brown coal": 0.003889,
+        "Sub-bituminous coal": 0.005555,
+        "Coke-oven coke": 0.0002778,
+        "Coke oven coke": 0.0002778,
+        "Coke Oven Coke": 0.0002778,
+        "Gasoline-type jet fuel": 0.01230,
+        "Conventional crude oil": 0.01175,
+        "Brown Coal Briquettes": 0.00575,
+        "Refinery Gas": 0.01375,
+        "Petroleum coke": 0.009028,
+        "Coking coal": 0.007833,
+        "Peat Products": 0.00271,
+        "Petroleum Coke": 0.009028,
+    }
+
+    # Fetch country list and demand base year from the config file
+    year = snakemake.config["demand_data"]["base_year"]
     countries = snakemake.config["countries"]
 
     # Filter for the year and country
-    df_yr = df[df.Year==year]
+    df_yr = df[df.Year == year]
     df_yr = df_yr[df_yr.country.isin(countries)]
 
     # Create an empty dataframe for energy_totals_base
-    energy_totals_cols = pd.read_csv("/nfs/home/haz43975/pypsa-earth-sec/data/energy_totals_DF_2030.csv").columns
+    energy_totals_cols = pd.read_csv(
+        "/nfs/home/haz43975/pypsa-earth-sec/data/energy_totals_DF_2030.csv"
+    ).columns
     energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries)
 
     # Lists that combine the different fuels in the dataset to the model's carriers
-    oil_fuels=["Patent fuel","Gas Oil/ Diesel Oil","Motor Gasoline", "Liquefied petroleum gas (LPG)"]
-    gas_fuels=["Natural gas (including LNG)","Gasworks Gas"]
-    biomass_fuels=["Biodiesel", "Biogases", "Fuelwood"]
-    other_fuels = ["Charcoal","Brown coal briquettes","Other bituminous coal"]
+    oil_fuels = [
+        "Patent fuel",
+        "Gas Oil/ Diesel Oil",
+        "Motor Gasoline",
+        "Liquefied petroleum gas (LPG)",
+    ]
+    gas_fuels = ["Natural gas (including LNG)", "Gasworks Gas"]
+    biomass_fuels = ["Biodiesel", "Biogases", "Fuelwood"]
+    other_fuels = ["Charcoal", "Brown coal briquettes", "Other bituminous coal"]
     heat = ["Heat", "Direct use of geothermal heat", "Direct use of solar thermal heat"]
 
     # Create a dictionary to save the data if need to be checked
-    sectors_dfs={}
+    sectors_dfs = {}
 
     # Run the function that processes the data for all the sectors
-    sectors = [ "consumption by households", "road","rail", "aviation", "navigation","agriculture", "services"]
+    sectors = [
+        "consumption by households",
+        "road",
+        "rail",
+        "aviation",
+        "navigation",
+        "agriculture",
+        "services",
+    ]
     for sector in sectors:
         calc_sector(sector)
-    
-    #Export the base energy totals file
-    energy_totals_base.dropna(axis=1, how="all").to_csv(snakemake.output.energy_totals_base)
\ No newline at end of file
+
+    # Export the base energy totals file
+    energy_totals_base.dropna(axis=1, how="all").to_csv(
+        snakemake.output.energy_totals_base
+    )

From f0bb45aa6c14ad9ec70b2bafd87904015c943622 Mon Sep 17 00:00:00 2001
From: Hazem <87850910+hazemakhalek@users.noreply.github.com>
Date: Thu, 17 Aug 2023 17:31:26 +0200
Subject: [PATCH 09/58] Update Snakefile

---
 Snakefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Snakefile b/Snakefile
index 87e02c84..d3d86f63 100644
--- a/Snakefile
+++ b/Snakefile
@@ -236,7 +236,7 @@ rule prepare_heat_data:
 
 rule build_base_energy_totals:
     input:
-        unsd_paths="/nfs/home/haz43975/pypsa-earth-sec/scripts/Energy_Statistics_Database.xlsx",
+        unsd_paths="data/demand/unsd/paths/Energy_Statistics_Database.xlsx",
     output:
         energy_totals_base="data/energy_totals_base.csv",
     script:

From 2b88844f9e957582c2cbb4299241ee9d070b4f8a Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 17 Aug 2023 17:45:10 +0200
Subject: [PATCH 10/58] add file with unsd data sources

---
 .../unsd/paths/Energy_Statistics_Database.xlsx   | Bin 0 -> 9202 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 data/demand/unsd/paths/Energy_Statistics_Database.xlsx

diff --git a/data/demand/unsd/paths/Energy_Statistics_Database.xlsx b/data/demand/unsd/paths/Energy_Statistics_Database.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..631ceabb698ebbe4e3f087edd45aa591e27e34cb
GIT binary patch
literal 9202
zcma)C1yo!~v&IH@cZc9^L4vym_u%d>L4&)yyF0-l2_D?t2_!gy;PQsB+3cVF-#f3*
znL9PpRdu_+n%iZgAPo)y3-UA~6CzZ95`SGFf$wh|jIA6Q75?IZ{>fv})!w%QoRb0r
z0fGCQhvA8{GVY^P4-<0G8;W5so)rucqk2yieOuBWpnXP*^wz8-IE}X#Vn_!1SQxxP
zMxBqh-By=pY-wt8Ey;TNIx~7)T&G@N>3G>M%kh{rsNHZWY68(r=a1h<^Ts8`zCAGH
z2t=t1%(rM49X=XB`yxde>@X#jCNae9j6H}j&1|!GgFEF4NtarbJ6)@JvPl@%6M=51
zVy2g-w7q5|xpWbA+|wb0e=bA*y#k;2R)r2Gqk(Z{_@kBtln>x4J~qC`N2yht=!NR{
zf^$i}7Y!{`Y|&Sg=5Clc54XWdjGfxEL=4UIek(k`;P^LWFhDqrYz-A0Z0&xd_HQ<D
zAe)=j-?09h;YkmW!O6_n+W2p_XY^YFg~%%aARx|oARuV}Wc$g?%;08ib*Q0byTpP1
zKpU{Cbxqliu!kASIFeL^fZ8NXWFDHNu9PP*DS5Bj5Dg9qzaP{AKfgp6WDU^NV_c~o
zoYmj=eF!Abv3kvsNO8#GWM=&V6{*W{_a*6<@;Raacmfm>>gZP0dxsGra)%aivM$RN
z90W#ZI&zFC-eEV9*4Ye+{hP*eHzfc$In^f;WLtU4X$rH17&-`4vX;;`)g0w^^QB1u
zcT%#u3~5Tb9f|=!cw;H+Lw^pnkxRa+<q}CzY$N(By%RL*RAg;R#1-~P>e=+!`0$J&
zLbMS5()Gngi*Nlx;wjGb3Y(`}DDF9Qw~H#CgFOkct|{J9?JwVquFK+4XP@-L7=~Fk
zW~h2vQg_7;zUfEcfIJlRq-F%@%>m^1Aye@pb!q!8at>yR+qx!fXQka_y&Vy{zL2k>
zc519^Szc|_Btxk5H^H{o2f4nlwz?epo(Z<5+ig4j{-!b#^%VdRAvVSC6sISl6r?Le
znQ@4{LE!6(*@s1?rKN-8Q&2Xm8FK8%3?Y!FU11h5%}a@P09F;;U7N%8+Lse%)`TUF
zc1|LcA@b^`7peLUuQ2e%=YD>-Ju&tonkbw11_pK<gH2Uue+ye&tWRyt>tT))X(D^9
z)9c6f5LnkcvbB)M!+v?6`!C_B3b=lZT^={LXT^+N4+qi5*$L~Pp@}A9B%H$W^T7%I
zE2X38{rY#e=1C0ocRqj;!>v<&n1*g=@0aODP}b{P!BE)1wM$nPIb=d+6@^=mT=1$7
z^~|WZW^i(Kc~|o(Q)Aomvn!gCOfz^yv{Y;Fa0uS8fqI)V+CsV1(W`g72H7Gaxw--n
z*{FH7puGnK&Vb~%dE<Ox@0n+W9ikN-W8jJiaD{YtJRrZ#w`W`&<ynuKQP1ssJ@_cq
zUjcG47>nZuK8B9y92<}d?v7S2z=;|>jBi&;JoQQ+AAy&&u+NbTt18wMOZa$V%3EuA
z<l@L6_p7q-QLVd~$i;O7);iN{4JY%i*nzkMtTcMQSCx~aIl6Mi?m#f_x*7Y*V9)jE
z8wLuD7F~uIo@peLJUz+29gmE97|OI0ynyibu4Ha!iTtTWj{NbawcYm}fM|M`Ad?RN
zTmGi6ud+mMA!cub5$|(_yP^p^3X>k|JGC1OwAL((4L*I2ybNG*DcOlwPB2GHUa#PU
zFyNL*O}f3~@?Fx_f!T%2SS9;7^FkRA3>qaU%Qv$=q}{;ydYB@Ep+aim8%m=8*aAs$
zQg$dv<P6~@i$lFC?WmMA^S6QI);tr^^@`7Fc(%nAS)lSWs;CDUH1o4yPyle8{s=co
z85Fry;^`%Xl{!&K1{OSLL*Km-R>+dGu~X`Vnan07mn19n#i|CHhL>1zq@EoI>dJ&9
z!*?Z?EwGFJczwhwLT!Q!QEjsj9GaNikoB2ChMARSQ=uT9<gW9~H(KTAO$8*Fia%=M
zS*n~YPG>2&gRc>a8zm;v*)n|w!%T|3?N#T?Qz_zM>{S4T*gKAIi$b{I+YPs0gJR5o
ze$$Z%hX+lVc*17JHnqT<9tsY>`aTR--K#Bria8uS<1#-g=@vd~mC;&4Jk8xD@bK7|
zcJy{%s?I@QZRvax4U_i}AU245sHt7zD~uKfzJ~}nKvLAvV?*7hYD}iEEsLjB9b}(M
z8yxV+ASwahGqn-^0S&7o2k?)%$I$=LewHfA0P)4k#Jnn^em?~tg+nT=KJ&^pS8syu
zyFfwnOu<6Y9`0?otJ}M&vibFlPIfAEbLnsJpBu!>F36j|yHI8v3obG7EZdFV<wS0n
z`Nt7`-B)g|nD33c0wpJ+zl?-dT`rW<O_SclmN_!(q{w4%fpXSw8zimB46DhiIvnZV
z!_M2wAXN73z_dQpi;J3l$k^Pv&1pj_&c;z<@r*tPiOT5cU4td=XHc)cNBHGSo)35W
zZk<~qP!JF<pld||9`1kp5<^=D<ENv&FjhzY6BByS<LAlS!-H;LjwOj{JBrXy>_}Hx
zPv5>DWY*3n$MoPqf=BIEtHon4y|l*Omq~X{xSKl(hta)&v9Jl4-VbVL*!@N?`BqP}
zX4K4T0Ff`Xk-aGaFL-QJ-AOP{K2O~9v$Ze>i8}21H6+HF2DuZwS|2s$=0xR3mWRPv
zXyYt-0ewAx0HmkBg*e$u8URt^#gOwm0lN=~7<=ov1F&56pqby*7~uyQ6$fkT-7xq9
z$vS>GK-_kY-B<{0JpkGnvvh-sixiL3HzgJ|?LHIK>EXj0I}ZnoS~*j^UOv`Cg1^P5
zLYK@CK)TI{YuekefBC{XHCZKXtw6Mej<O+b+#yY6sx2Cq0pGNgTka<L2f4ZB)!EV^
z(l0ds?NJe*U8Sq7gQcUHv9Z%{_xz0floVZ&84&qOAoAG%;{7SY@??O>I9d5_CitL7
zsRW)Oj?Uuu&}<}J#3T^Q_0!h*6!wYD5%-?U5pu)~%XfXJhFx-D>GH=tP(4P>2@cHZ
z(x4yptcA5cH_hF~(^8{;Q?LvMm1+qlw=FXWwom5<LDeQwq~^)<E>vbAaLY8A7hacn
zH(_ajqm_6A-8w3|TKeU!cW(Q8K|<*AD<VjaNcI{rRySxTUlF@Bx0~sOSF&uj5AV*n
zZGs<qa*lCoYey$v2^XH2{FsM~@YtIFvAZdg9-7%p)ZRpC@F~b7;+F8dde_pqGDR8V
z`rxH10CC&Ocb95%-96`?fNMF?fk8#3WUFzpZtFBwB%mmvDV#a8{utBcn>bxx(*m)L
z_^J9M>3zFrHr7WkOSAL!uK-shy``utyo~y}i7#cV)g(Bp?1m;oXNFUalzf6n_SIT$
zleLW`Fi^4#%?YH0AulNp9DB|YP;;}D`=$N4%9N;aMbdh>u1?EU-u$xUvvoN5pfvD-
z@n{O||7x9|kxyotmTkTSa^NHErWcr3@W6#xo>{|L&7@iN7(|mHx*;8L{9xA_B#(wI
zk*X3qV-$?+Os>~n{_^2W|M4No_0DbXx2S#P<!$rLXy+#DsObIkG5%LRRFg*7<y-H$
zoy5E3-rUD{FIVWcKA6vY@>6cyY}LqJZn?FN)a9<;zQ?s`jpq+yZL6{^XU4yXUa;Hq
zYRqJ`KuR5UOm9?ilo>;@@sx_NX|lBNS(_QF<d51V?%-E8RNy~IpE|*;tfbypEc5Yr
z<yF#lfiA?$7Hsl1^Iego)|$6FuZ!0YJ80K?ov&ry;e12_SMN)2ZN9o}$wIrLgk5Tz
z7-N!JtDZe}UZksB@)D-yKQ6xLcEp3|SiKg$tdl!qTdJtL)L)`pK0PO)=vb?qlHa@-
zT`(Wu-K_5Gf>%3^|90m7wYB)$`jAUlQp8O6@x%K5M6lth?fJeX|5!X)gVlYN$E%1g
zXU49p9ee&DTAAj@Z=<$N)tBBS-FLw?oRc!Kdq+&e)sjmx9I$6<UfGX`Tl;K&#%b+r
zOZHuI*3Cu3nd5a#Q<hvx{78m#4_>c`s@zYyuUc>zY1Z;TODne^t6;m6HSh#y=*~oc
zGBF)FvtCm(Y}EL%GW58tKRE_E3cB`!Ty}^5OC(ITDPQC@+sesX$_(sJK3&$|RW7Wh
zTyX4N2}!EOLM~A>6i&!+;z)%@C8t$TaR#}f$JNNtrGxSY6o`ezS<nZV=y1|Vg~ud^
zRZ#H;$BDaZjj+Su!Rud(u)q&cX|T|V1_5wLFG>Ie{-%oJD4Plr2a3T$ZXEPfLa^wl
zw+#X$y!lD!BowfrnVNaXSt_D(o9a=D22Hwscj9nk)&70Ter(`nu;ZleNf_icVSu{6
zB8&mmLB~qs2^C^p;{z=}-$dRmSX6^RG5T6(fTAi(hxBHJz_`>2L*5hZUn#IibHcEx
z$_)IzQeYiOqpAUEp(=qLAT9Jn=T!+D0BHxc({@_P<-`FU3{N#uSB$Z?xE`I*<(XPn
z3I{dTTMbR&@a38DNm5t#pEE7vvwf>T=8>~1a(f`Z^T89d%yC>ZZEPEEJCOG;8WKpe
z|9@!SV;WRf42QX%Rh0pq?z1XVdo>bQ>jRy@<;GuAeNR(E+A=wnl*F#|6)%97v9_?D
zCzjeL(Wl*42_0zyck@hhv(D&N7x=oa&1ayn##5EY)XDXvju|d*e#e)Dg{dq1AnBV&
zof0n~R3=owjbSQQz&(-%kQFKe5@_ZVA<Cxini3K-A<DZ3^dZV>Ck-(%WhkXp5sk!(
za9K+1Fr~P$nmwZVD5Zhh=DC%V7A0w<7ge+EWfSDSdx41yS0D3GasZB{X%GsLk(Qw4
zGxN=u?NH&ABr0Tz;lRmSdwMq{Oj(06v7)5YccQGRWs)*86edBq)b6PWp*fWO!>5it
z+rz`;y10I+TbgYCsi`93@0{p>q(pz5T6oYuKDD@t<mC%;C)IfM6y)bxYvByz^H)dc
zn^4w{>#<UQZ~lr<pUy^pj?)ywUFETrQYu-%D#0BqyukX}RHB461W2y1YM4q^)DLjS
z_6ORg*3^^FYgi$Q61tXQwh70cX{tzr<%oiW@gAwrqbh8-o6N)^o1yrTMwhRVf(NcQ
zmM1|7hOe~-hLlfk&e@Q-A(T`2O&BauiD=Vf-ind9y=kJe*75;O0@EH2D5Ur}$#AgU
z-PfoyKV)j7qlL<U2*$5}yvTBGE_v4ghj#^5W~>kQ=6hLhY|I{J49%8>y-Wc&sv^Nf
zG}y%;LA{?7xpynT`*SnR3K%~{H4Q;%wj8~Th<2hJ^9z3n>0p9<U%u<1oXujj9(<TY
z8_+}+?-Cuz`@6jiGQUasI}liJqIHmjg&8D({7n91;~r7^Rt)}~%`CFf<!-^*Z7t8p
zv435#4Sd{c)J5mwjW;(etcwn3CxJ}rs*g(BHPnB46nJEd1sq@=UI`BZg7EC`9i7~*
zj2)kxe6O0d?IIKM18r=V5)`bCV1R`{ERC~Lq03}vS`UosecVeqsLYEs*Y{|svo87Y
z4aEAlZ!eb}O>Z|Ns%QP7<dv~x2BDqDgwPdm$^!=Jgx(NhW=f-@2Mq++QJ4pCGzh50
z+v{^=tX5O8gMti@3eg&VN9hx7&+|t&r71==Q7i;7Gfm?#ILJyWs*`=RXM|_0A(kPH
zX3})MCHM9tQ?F3TRdmFp^;1c!OX(EB-6Wy3k0jUq)Gj2Be6PMk;5!nkFWr~Az@<us
zeEO9#UMij<IEH|k4QEJ^E`Zu!J#kWmO2mytF42nIi&)QDJX+B!8IA`)EvgIWaDmT@
zf*4oT7=fY2%_I=5Nf9GKS~8b{GMbc@`@w_Q|1>iBvLX$xR1qCsU6bU3A!^SIs=lXs
zYtITNx0lpuEO-3{h>I-fS;W~PSSA@`N`(F(!i)pxyPQ3KD+h^~03Afl=0d%X+Gki(
zGV!uaoLu9r3K<khZ#XRbvp;QZ`Cv@5Bkw4FgFv+BQD97qcqxmJVwqHFWvLu>Sio$T
zMY$CaD<k=-^4;PlSCU^j3JMD?*H%}h#an$J2AXos>-niq-?+IwaeYljJmul4?>r(@
zpyCLXMvv)N=bX~rJ(uUq*DZTx4TePYnhaI~zm&BS_94-E<Bb?L=?h1<$<gi;Kv|b?
zwr_L{w^0$hk8&!#u;0S1%BnLSk!M?9lQRCASwyYH8AkMSs36B#QGyR%RjS@ppjMm%
z5_BetP`9{v+f7l<w#dt;8YfX1&AUSRJSq;>lPkLS>)j*M9zl)H`@wgoJe=putQW=X
zymtc~=^N?kw|0oZRu@Vc%-V?ys`P86+gj$`HX1B8DHScKp5-@I)roJL!oW@H<XmmT
z`*f*XJ3z#QloTD#YBD1frXEGmRKqq@_oMizmCV`iwQi*jT3xYp(w9Cpb5OB|k4Nu+
z3*NZ%nAq12ni`UO%s}Ms{^+!db8b3y{gxTtbK{`ctE0Ki^5aHOC2d^d)U6n1?(By)
zYn}DSjhmxa04qqp=0{ik^&M%$?{NS53>g_N)JOqiW=vo?>Zuf^C2VWsWNhQ~M%mrY
z*iq-{Ng7ceviZb>e3v*dN5gDaQWo|hH`*{<`3AJAg9J0m+8qnlXH8bI0JacSMR0R-
z?qGCSQ&)bst<P9X!t5PmZ!>l%d^=*_jSBT@^4N9eKuiMXX8=;+Jb2i(RqL|&hX_Z$
zq&&*7OU1mZ+}G*>&{?rG1!5?+Q&M7ua~1#x7Tgsn0$#u33{6Dl^jAWK98q<_5AB5v
zfngIGl_A!Hm7L-Q*PQyXADo%hmqIx4iR+39>d(J0g$ZTl*k#Ngy*y52q|^ZSslzP$
zY{4XGiz`RXM5zF231Xo=F(R5@aXiAHoQE~TRKEQ38fWr`EWp{G`Cc_w>ydKJkE+f5
zEyXL!-E+Y^34Mu1@S9WrK(a%E^%(DHZG{$9m|;8$n?rqYe(;AcTjQDB<O7BVY_S9-
zX_*Yzq6?5&^$uoQ3%)NUGfI&b>u%>v{s@R~jk~f#fY5UTk1%8)^nWLc`gV3txMwQq
z$Zr90r;W{yZdHcP{mjgbK|t(Rr6}#Ln<2Z^T}$r?b9jH_h)PjFtb1hb>cW<FU(@_*
zn$nU<<j`t#)YLI4SEXv)CA+@(u)3Z$Nfb=W*B;Z+rDve3P#tQQIi@d#0u(P9;Y$VS
zcj?jmvYH?+Y54T8LO6B>sqU<{x@lP343F{&)HlPs$?bM!D&cL16E9A6k9~Si1kB**
zM(@mGBu}S2^Dz|mUU%;EH>MNF@It@p6-8al@qkN<a$vY(!@vVTs!1v)>5;-dfQIk{
zxM*6AlSoaB*9nR;7%?K(T2q@MKp-Sd68~6BU{IIepJ3p4r5H2D(B20do3~CjAE<Q1
zHjCvjfud+9wVfY}o~^lT!Gi6RYzciHxpnkS-rJKWeRl17dd=MYwD(4DbE-aCPe0mE
zy4Il)$}^z}l(%ZK-N)!sbp(P4c2jU^?rOW~2ogFNBg5Z7N$>-ywYByHbV%E@G84Uz
zir<PMvt1|VqSnHXHNm2_@s)uXxCi@-??k^In13gvf0cOuTY+7ZcS&jwXjD<4adDr0
z(=*#s_G(~jYx!j1pY`^}=X$$<55@3GQDu3El?!8djanrUa+<l)ds!h9)x$MBPby5h
zgIrxyp#e`MmfNkl@o=PwY4_rs0SDx-7Pj1TG;Y<pM_B^MmZna}1T@&O>u&R12GuY5
zNR|Mop&Jr@#)xBziq`hy3D>?1D{QY2)25i;vwXeD(@HST;{SY=4k}$#;$#(z@%F}z
zBz%CpxN-m$RfHX~UG!^uQTgfi{=RS2<H=_YkWDM}o_cD5RcxgPADI?hVO4Q9Tl^`t
ziCGf3D}2W;!fwOpB{i98jV`Sj$GDt~*%2tCyio)segp^RhhW=ZKtAV#$Y@+@A;44T
zh8+Y1{TU%gGkphRBPAyXa~sp2PW+4JvBM%4vJVCVk<F0_mEK~JB<|Of7cmXwOp=Rl
z51`#Ya$5~18bRASjycG?Xa+l2gY&T%X0p4!aj6+!yfH7Fv2}I6I3fuhL`~RHY@iXW
zd`&`}0^gF-L_>58$rwc|f=45SD-Bxt8Q1BH&aI54a7EF0rIeA3hi0`Url63sY!UGR
zKWizf#X)}BoO79P5H}c;q_!;wuK14thu0)BCZFc3JFQeDD6f*?J<^kQBzwIBn#N>0
zyj1&%u7y2&dN(U<IxY7DNI2iZq|NKcopyU5`s0rCUo6416$H()#6}-{vgpBl+rg+t
zn#8uuHeC`$xcb9-3^W>{1XG&fwm9>Ykxg>V;0!}$hs7%MlE*c(-9Bc^+Po<Fn$?oM
zyt-U`hI3}?PNV4+t24vS9GhP)tf2E;v2y*MDe}e0m1U)z@c2RS>n*zMy#yAXOBY=B
z5(?!-f{$pp@fb6Z5~p0p_aDtkMR8pHzaOoMFC5XmWOpf#C7EH_0SgQMQ3|21!(FV2
zg;eCZKO}YFAn6SyjG0paxuM7BvgR4UzI{RB0#bL3BGG1YnERTyPr8d~$Ggw(5&s%|
z$_NG;Dv53*WFDNk$V$*aI<|m@aCh4f$5hN}0S?DVO1;935Yx6ZmvcwSrM=)qDR?sX
zo<~SsdCUmjR!^4)cr(`P-VE$U*9`=*9#?T-EkzdxqWuMyK%evn=T@q$nkfE+u)#nq
z0?#c`dO3!jo{6aaYj7oXErSFzdzQpN0diNrt3IqAuPR@hIP-8Pu^nPr2v{_H6a^*T
zlTJ?B8_)HmQZ4>{>^4r6AMRGUw4==$YN0i`>{yHATtP8RX$F&~k?~;ZHsp!43gnTc
zOo$S_Eu2e_%OcaF?Jld-8)mrb@AAevi(T*og<FZP3>pL>nQ_`qCOO|Vxe$LGlvHTU
z3*Dp$Q=`(&=Mp-F>}3v%x2T^m%Yjy4d)+Q_3fpUmE>5+Y!zJd23u*yTf|4WV`=7*w
zNTkVnJU~@3s8>r*L5REEIyD0?h?~wY_vk&gP=i*H+)ebL*o&oN{za?ozDGx`x0S5z
zO#A5H;V<YMW9JDx;yd1ls43F3K?)<<C4C@R>ZT`S$ii^z4rDBWB`6&5SS=`H%!s+s
zvi&~&vNJU<G;%WDya*w?FQzTyg{S=(+-z=KuyY4`?!E7(OYibAO~CQ&O%3%rA9v5C
zFSQvK;%NuxvQy9QCrN;p(;-q1BTlwG9=F^FDjeaYoZH8<=uKvMsguWiD+@9`__yKx
zN<!aEn-HoCVl6xw^R8w*zKk4Zmdn`nO;vmE3yitRP7R1OT!pqU=t~~IGh;C>2Ba=k
zYu9jfdzu+J@!gGp9<Zb=H$giVV_<d_2Cvp+?nm?rU|dtqxXx=5jSbLj(s5Hv`82ya
z`aT@B(Lqriwei%7&jeLBp)@)bX9d?a&59v{#n{8ZEMH)EBk@pBN;k)(j#@JH3ZE6#
z+=g&6vK2}N;drJwJfs=~kSD!nPu$e-7}MTvk<0Kz(XJZPTZk@GS6Tdo#$Hmp=s1nR
z)B<5FIX4=v%&&A0;mG9TP|ZXi0YPYcwCcN~@A5UAQh8z_0<JC(iNOX(ro9D%r#7Bk
z`-^q!>NFpvoX}P%XRdveZC!sly50Z_SbSGL{fm~pP|lV;yqYS==`?r{A!~9iUiOXm
z(cxOyoU7a4Y|dzY)Kjlmb9kMC``N`np9KjnTbEspWfk)f3e6B&NDm;14*G^!quiY|
zNt~TUW5v9DsH*$ER+_>qx8lMY%^Ghak65%)MudT}%-#KtdrdqgVTxL9dAn$qnrT}|
zX=sS)K$Zf_jn*Lj%i9A_V*b{qFi%VY9m}bQKVruBTV`iMz`%tKSZ76gKDqzt)goU%
z$BP(4mvK-4K!D($v}l=Tfg)GBn=l<dPaOf6&ziFW#QAi*Y2SWk<_|0TGW3`J&XZ<h
z@q#iiUzedvFC!i*)(BaoGIllXjM<nxAjPrHG%OGFO^5x~9S^bTJF1IcZ+}OnIf5*L
zjN}Zg#kiC>eC5yxV9<tHr^P6==Z&IaVDE5BS$sTa5+!kTV|G{ayGW+QM8D)}r!_wO
z`c_o}kS<MVkbYAdE5*c((<dyIVE*x1jf5|UL&$IKweDPY`=^8#qWP8siK3_<#Dw;%
zJaV=h5;aqjCi>xBU5^&9C6_iT(K~x)Oa_gnEsu#W3r*SZdOvbpeuuW~G-N~UoQPCw
zs(B!Hp|MwNLx*{gQb>|O*|*s{&LPk3&fYsO;_fJy<LO!@-=%q@p~xo1M|L+!6yyq-
zTO!Z*2vCp)1w#k<-#hw1%z@)6W_%jI!jE4v&+hfl4*xUn)0O;hnV<2;|8=vc0Oxr;
z-Ha7z^S^`C|4ZztH~u`H#593oKRe~W7Wm%-sow&g&jP<X=fA|B_w{G*{I44U-}3$`
z_Pdk*d!f$?{_M~ER^S8mkNy4X*8CR#_p+ZF63^plpCiye_xT^~iT_;tpACv%(jXw$
z|C0UPuK2xz=MDT>NdK)R8TgX{BK?mQ{M7QZtp2B#KZ_^7wR{6^2B_tCDdkVmKXb<4
zqV&K(?!RP^e=7Jh^ZBiy1mhnj_$v?kE&Xo`{O1o;7whNB|N8^+XGH${rh)$y`{zma
z$Mf+F|DSQ|Z#~7prqdH}e@3ytH}I_C&-3%Q27SW64frcZP>=>Z{fP(IN<;yHhX4T?
ICwltyKSdX)dH?_b

literal 0
HcmV?d00001


From 576a76e56e6546edbaf1b6a450438b621ec004ee Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 17 Aug 2023 17:46:11 +0200
Subject: [PATCH 11/58] add script to project energy_totals of the future
 scenario

---
 scripts/prepare_energy_totals.py | 80 ++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 scripts/prepare_energy_totals.py

diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
new file mode 100644
index 00000000..3ad1ab79
--- /dev/null
+++ b/scripts/prepare_energy_totals.py
@@ -0,0 +1,80 @@
+import os
+import glob
+import requests
+import py7zr
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import country_converter as coco
+from pathlib import Path
+from io import BytesIO
+from urllib.request import urlopen
+from zipfile import ZipFile
+import sys
+from helpers import sets_path_to_root, three_2_two_digits_country
+
+def get(item, investment_year=None):
+    """Check whether item depends on investment year"""
+    if isinstance(item, dict):
+        return item[investment_year]
+    else:
+        return item
+    
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from helpers import mock_snakemake, sets_path_to_root
+
+        os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+        snakemake = mock_snakemake(
+            "prepare_energy_totals",
+            simpl="",
+            clusters=10,
+            demand="DF",
+            planning_horizons=2030,
+        )
+        sets_path_to_root("pypsa-earth-sec")
+
+    base_energy_totals = pd.read_csv("data/energy_totals_base.csv", index_col=0)
+    growth_factors = pd.read_csv("data/demand/growth_factors.csv", index_col=0)
+    efficiency_gains = pd.read_csv("data/demand/efficiency_gains.csv", index_col=0)
+
+    energy_totals = base_energy_totals * efficiency_gains * growth_factors
+    
+    options = snakemake.config["sector"]
+
+    investment_year = int(snakemake.wildcards.planning_horizons)
+    demand_sc = snakemake.wildcards.demand  # loading the demand scenrario wildcard
+    
+    fuel_cell_share = get(
+        options["land_transport_fuel_cell_share"],
+        demand_sc + "_" + str(investment_year),
+    )
+    electric_share = get(
+        options["land_transport_electric_share"], demand_sc + "_" + str(investment_year)
+    )
+
+    hydrogen_shipping_share = get(
+        options["shipping_hydrogen_share"], demand_sc + "_" + str(investment_year)
+    )
+
+    energy_totals["total road"] = (1-fuel_cell_share+electric_share) * efficiency_gains["total road ice"] * base_energy_totals["total road"] +\
+        fuel_cell_share * efficiency_gains["total road fcev"] * base_energy_totals["total road"] +\
+            electric_share * efficiency_gains["total road ev"] * base_energy_totals["total road"]
+    
+    energy_totals["total domestic navigation"] = (1-hydrogen_shipping_share) * efficiency_gains["total navigation oil"] * base_energy_totals["total domestic navigation"] +\
+        hydrogen_shipping_share * efficiency_gains["total navigation hydrogen"] * base_energy_totals["total domestic navigation"]
+
+    energy_totals["total international navigation"] = (1-hydrogen_shipping_share) * efficiency_gains["total navigation oil"] * base_energy_totals["total international navigation"] +\
+        hydrogen_shipping_share * efficiency_gains["total navigation hydrogen"] * base_energy_totals["total international navigation"]
+
+    energy_totals["district heat share"] = 0
+    energy_totals['electricity residential water'] = 0
+    energy_totals['electricity residential space'] = 0
+    energy_totals['electricity services space'] = 0
+    energy_totals['electricity services water'] = 0
+
+    energy_totals = energy_totals.dropna(axis=1, how="all")
+
+    energy_totals.to_csv(snakemake.output.energy_totals)
+

From cba9e756a06013f2d0607dd64d7e74ba74e723d5 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 17 Aug 2023 17:46:48 +0200
Subject: [PATCH 12/58] add necessary csv files for the ser to adapt the
 activity growth and the efficiency gains for the different sectors

---
 data/demand/efficiency_gains.csv | 2 ++
 data/demand/growth_factors.csv   | 2 ++
 2 files changed, 4 insertions(+)
 create mode 100644 data/demand/efficiency_gains.csv
 create mode 100644 data/demand/growth_factors.csv

diff --git a/data/demand/efficiency_gains.csv b/data/demand/efficiency_gains.csv
new file mode 100644
index 00000000..557c51f4
--- /dev/null
+++ b/data/demand/efficiency_gains.csv
@@ -0,0 +1,2 @@
+,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen
+MA,0.91,0.82,0.79,0.88,0.71,0.91,0.82,0.79,0.88,0.71,0.91,0.82,0.79,0.88,0.71,0.91,0.82,0.79,0.88,0.71,1.0, 0.9, 0.87, 0.99, 0.9, 0.85
diff --git a/data/demand/growth_factors.csv b/data/demand/growth_factors.csv
new file mode 100644
index 00000000..3eca6c46
--- /dev/null
+++ b/data/demand/growth_factors.csv
@@ -0,0 +1,2 @@
+,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas
+MA,1.5,1.6,1.3, 1.4,1.5,1.1,1.2,1.3,1.23,1.52,1.13,1.4,1.23,1.43,1.231,1.132,1.41,1.23,1.32,1.40,0.91,0.89

From 95b288e46e790c90be0c81b3e4fda1257dd45332 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 17 Aug 2023 17:47:48 +0200
Subject: [PATCH 13/58] change locations of the unsd data

---
 scripts/build_base_energy_totals.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index 8dcc34cc..25ea970c 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -248,7 +248,7 @@ def calc_sector(sector):
     if snakemake.config["demand_data"]["update_data"]:
         # Delete and existing files to avoid duplication and double counting
 
-        files = glob.glob("data/demand/unsd/*.txt")
+        files = glob.glob("data/demand/unsd/data/*.txt")
         for f in files:
             os.remove(f)
 
@@ -258,12 +258,12 @@ def calc_sector(sector):
 
             with urlopen(zipurl) as zipresp:
                 with ZipFile(BytesIO(zipresp.read())) as zfile:
-                    zfile.extractall("data/demand/unsd")
+                    zfile.extractall("data/demand/unsd/data")
 
-                    path = "data/demand/unsd"
+                    path = "data/demand/unsd/data"
 
     # Get the files from the path provided in the OP
-    all_files = Path("data/demand/unsd").glob("*.txt")
+    all_files = Path("data/demand/unsd/data").glob("*.txt")
 
     # Create a dataframe from all downloaded files
     df = pd.concat(

From 509cfbf6ef20bdd3cceeda3dfa6bbca19c6f8e4c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 17 Aug 2023 15:48:06 +0000
Subject: [PATCH 14/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_energy_totals.py | 75 +++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 25 deletions(-)

diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index 3ad1ab79..0d9f2907 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -1,25 +1,29 @@
-import os
+# -*- coding: utf-8 -*-
 import glob
-import requests
-import py7zr
-import numpy as np
-import matplotlib.pyplot as plt
-import pandas as pd
-import country_converter as coco
-from pathlib import Path
+import os
+import sys
 from io import BytesIO
+from pathlib import Path
 from urllib.request import urlopen
 from zipfile import ZipFile
-import sys
+
+import country_converter as coco
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import py7zr
+import requests
 from helpers import sets_path_to_root, three_2_two_digits_country
 
+
 def get(item, investment_year=None):
     """Check whether item depends on investment year"""
     if isinstance(item, dict):
         return item[investment_year]
     else:
         return item
-    
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from helpers import mock_snakemake, sets_path_to_root
@@ -40,12 +44,12 @@ def get(item, investment_year=None):
     efficiency_gains = pd.read_csv("data/demand/efficiency_gains.csv", index_col=0)
 
     energy_totals = base_energy_totals * efficiency_gains * growth_factors
-    
+
     options = snakemake.config["sector"]
 
     investment_year = int(snakemake.wildcards.planning_horizons)
     demand_sc = snakemake.wildcards.demand  # loading the demand scenrario wildcard
-    
+
     fuel_cell_share = get(
         options["land_transport_fuel_cell_share"],
         demand_sc + "_" + str(investment_year),
@@ -58,23 +62,44 @@ def get(item, investment_year=None):
         options["shipping_hydrogen_share"], demand_sc + "_" + str(investment_year)
     )
 
-    energy_totals["total road"] = (1-fuel_cell_share+electric_share) * efficiency_gains["total road ice"] * base_energy_totals["total road"] +\
-        fuel_cell_share * efficiency_gains["total road fcev"] * base_energy_totals["total road"] +\
-            electric_share * efficiency_gains["total road ev"] * base_energy_totals["total road"]
-    
-    energy_totals["total domestic navigation"] = (1-hydrogen_shipping_share) * efficiency_gains["total navigation oil"] * base_energy_totals["total domestic navigation"] +\
-        hydrogen_shipping_share * efficiency_gains["total navigation hydrogen"] * base_energy_totals["total domestic navigation"]
+    energy_totals["total road"] = (
+        (1 - fuel_cell_share + electric_share)
+        * efficiency_gains["total road ice"]
+        * base_energy_totals["total road"]
+        + fuel_cell_share
+        * efficiency_gains["total road fcev"]
+        * base_energy_totals["total road"]
+        + electric_share
+        * efficiency_gains["total road ev"]
+        * base_energy_totals["total road"]
+    )
+
+    energy_totals["total domestic navigation"] = (
+        1 - hydrogen_shipping_share
+    ) * efficiency_gains["total navigation oil"] * base_energy_totals[
+        "total domestic navigation"
+    ] + hydrogen_shipping_share * efficiency_gains[
+        "total navigation hydrogen"
+    ] * base_energy_totals[
+        "total domestic navigation"
+    ]
 
-    energy_totals["total international navigation"] = (1-hydrogen_shipping_share) * efficiency_gains["total navigation oil"] * base_energy_totals["total international navigation"] +\
-        hydrogen_shipping_share * efficiency_gains["total navigation hydrogen"] * base_energy_totals["total international navigation"]
+    energy_totals["total international navigation"] = (
+        1 - hydrogen_shipping_share
+    ) * efficiency_gains["total navigation oil"] * base_energy_totals[
+        "total international navigation"
+    ] + hydrogen_shipping_share * efficiency_gains[
+        "total navigation hydrogen"
+    ] * base_energy_totals[
+        "total international navigation"
+    ]
 
     energy_totals["district heat share"] = 0
-    energy_totals['electricity residential water'] = 0
-    energy_totals['electricity residential space'] = 0
-    energy_totals['electricity services space'] = 0
-    energy_totals['electricity services water'] = 0
+    energy_totals["electricity residential water"] = 0
+    energy_totals["electricity residential space"] = 0
+    energy_totals["electricity services space"] = 0
+    energy_totals["electricity services water"] = 0
 
     energy_totals = energy_totals.dropna(axis=1, how="all")
 
     energy_totals.to_csv(snakemake.output.energy_totals)
-

From 86898d074cb4f2bac74e42e69d1b44c70eaa166a Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 17 Aug 2023 19:52:59 +0200
Subject: [PATCH 15/58] adding the excel file with all the paths

---
 .gitignore                     | 6 +++++-
 data/energy_totals_XX_2030.csv | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 data/energy_totals_XX_2030.csv

diff --git a/.gitignore b/.gitignore
index 9fa19053..4229db5b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,9 +37,11 @@ data/industry_sector_ratios_NZ_2030.csv
 #/data/Industrial_Database.csv
 /data/retro/tabula-calculator-calcsetbuilding.csv
 /data/nuts*
-/data/demand/unsd
+/data/demand/unsd/data
 /data/ports.csv
+
 /data/airports.csv
+/data/energy_totals_base.csv
 /scripts/old
 doc/_build
 /cutouts
@@ -58,3 +60,5 @@ doc/_build
 *.xlsx
 *~
 *.pyc
+
+!data/demand/unsd/paths/Energy_Statistics_Database.xlsx
diff --git a/data/energy_totals_XX_2030.csv b/data/energy_totals_XX_2030.csv
new file mode 100644
index 00000000..4e454a61
--- /dev/null
+++ b/data/energy_totals_XX_2030.csv
@@ -0,0 +1,2 @@
+,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
+MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.693556480000002,1.40252325,0.1264536,0.0,0.0,71.27159790000002,0.0,0.0,0,0,0,0,0

From 80ff049ebb391f9fc4133a9d30b674072267e9ce Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 17 Aug 2023 19:53:30 +0200
Subject: [PATCH 16/58] add rule prepare_energy_totals to snakefile

---
 Snakefile | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/Snakefile b/Snakefile
index d3d86f63..2079e706 100644
--- a/Snakefile
+++ b/Snakefile
@@ -240,7 +240,16 @@ rule build_base_energy_totals:
     output:
         energy_totals_base="data/energy_totals_base.csv",
     script:
-        "scripts/build_energy_totals.py"
+        "scripts/build_base_energy_totals.py"
+
+rule prepare_energy_totals:
+    input:
+        unsd_paths="data/energy_totals_base.csv"
+    output:
+        energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv",
+    script:
+        "scripts/prepare_energy_totals.py"
+
 
 
 rule build_solar_thermal_profiles:

From 6816947d99ab9c159823f4d6d401153f8d539da3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 17 Aug 2023 17:53:57 +0000
Subject: [PATCH 17/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 Snakefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Snakefile b/Snakefile
index 2079e706..5688b384 100644
--- a/Snakefile
+++ b/Snakefile
@@ -242,16 +242,16 @@ rule build_base_energy_totals:
     script:
         "scripts/build_base_energy_totals.py"
 
+
 rule prepare_energy_totals:
     input:
-        unsd_paths="data/energy_totals_base.csv"
+        unsd_paths="data/energy_totals_base.csv",
     output:
         energy_totals="data/energy_totals_{demand}_{planning_horizons}.csv",
     script:
         "scripts/prepare_energy_totals.py"
 
 
-
 rule build_solar_thermal_profiles:
     input:
         pop_layout_total="resources/population_shares/pop_layout_total.nc",

From 04c0c6fe98b7f5b918f3cf6ae5ae048555103f6c Mon Sep 17 00:00:00 2001
From: Hazem <87850910+hazemakhalek@users.noreply.github.com>
Date: Thu, 17 Aug 2023 20:59:33 +0200
Subject: [PATCH 18/58] Update config.test1.yaml

Changed the demand scenario for the test config to include the energy_totals scripts in the test workflow
---
 test/config.test1.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/config.test1.yaml b/test/config.test1.yaml
index 3d01c97d..ff323d3d 100644
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@@ -23,7 +23,7 @@ scenario:
   sopts:
   - "144H"
   demand:
-  - "DF"
+  - "AB"
 
 
 policy_config:

From 889de1a26b9a436fecf950f980f194501d6b37eb Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 23 Aug 2023 20:15:34 +0200
Subject: [PATCH 19/58] adapt residential sector and fix bug in road transport

---
 scripts/build_base_energy_totals.py |   3 +-
 scripts/prepare_energy_totals.py    | 101 ++++++++++++++++++++++++----
 2 files changed, 89 insertions(+), 15 deletions(-)

diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index 25ea970c..0ac7e30d 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -173,7 +173,8 @@ def calc_sector(sector):
                 energy_totals_base.at[country, "total rail"] = (
                     df_sector[
                         (df_sector.Commodity == "Gas Oil/ Diesel Oil")
-                        | (df_sector.Commodity == "Biodiesel")
+                        | (df_sector.Commodity == "Biodiesel") 
+                            | (df_sector.Commodity == "Electricity")
                     ]
                     .Quantity_TWh.sum()
                     .round(4)
diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index 0d9f2907..c4db7610 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -34,16 +34,16 @@ def get(item, investment_year=None):
             "prepare_energy_totals",
             simpl="",
             clusters=10,
-            demand="DF",
-            planning_horizons=2030,
+            demand="XX",
+            planning_horizons=2038,
         )
         sets_path_to_root("pypsa-earth-sec")
 
     base_energy_totals = pd.read_csv("data/energy_totals_base.csv", index_col=0)
     growth_factors = pd.read_csv("data/demand/growth_factors.csv", index_col=0)
     efficiency_gains = pd.read_csv("data/demand/efficiency_gains.csv", index_col=0)
-
-    energy_totals = base_energy_totals * efficiency_gains * growth_factors
+    fuel_shares = pd.read_csv("data/demand/fuel_shares.csv", index_col=0)
+    district_heating = pd.read_csv("data/demand/district_heating.csv", index_col=0)
 
     options = snakemake.config["sector"]
 
@@ -62,8 +62,81 @@ def get(item, investment_year=None):
         options["shipping_hydrogen_share"], demand_sc + "_" + str(investment_year)
     )
 
+    energy_totals = base_energy_totals * efficiency_gains * growth_factors
+
+
+    #Residential
+    efficiency_heat_oil_to_elec = 0.9
+    efficiency_heat_biomass_to_elec = 0.9
+
+    energy_totals["electricity residential space"] = base_energy_totals["total residential space"] + (fuel_shares["biomass to elec heat share"] *\
+                                                                                                  fuel_shares["biomass residential heat share"]*\
+                                                                                                  (fuel_shares["space to water heat share"])*\
+                                                                                                    base_energy_totals["residential biomass"]*\
+                                                                                                     efficiency_heat_biomass_to_elec)\
+                                                                                                    \
+                                                                                            +  (fuel_shares["oil to elec heat share"] *\
+                                                                                                  fuel_shares["oil residential heat share"]*\
+                                                                                                  (fuel_shares["space to water heat share"])*\
+                                                                                                    base_energy_totals["residential oil"]*\
+                                                                                                     efficiency_heat_oil_to_elec)\
+                                                                                             +  (fuel_shares["gas to elec heat share"] *\
+                                                                                                  fuel_shares["gas residential heat share"]*\
+                                                                                                  (fuel_shares["space to water heat share"])*\
+                                                                                                    base_energy_totals["residential gas"]*\
+                                                                                                     efficiency_heat_oil_to_elec)
+
+    energy_totals["electricity residential water"] = base_energy_totals["total residential water"] + (fuel_shares["biomass to elec heat share"] *\
+                                                                                                  fuel_shares["biomass residential heat share"]*
+                                                                                                  (1-fuel_shares["space to water heat share"])*\
+                                                                                                    base_energy_totals["residential biomass"]*\
+                                                                                                     efficiency_heat_biomass_to_elec)\
+                                                                                                    \
+                                                                                            +  (fuel_shares["oil to elec heat share"] *\
+                                                                                                  fuel_shares["oil residential heat share"]*\
+                                                                                                  (1-fuel_shares["space to water heat share"])*\
+                                                                                                    base_energy_totals["residential oil"]*\
+                                                                                                     efficiency_heat_oil_to_elec)\
+                                                                                                +  (fuel_shares["gas to elec heat share"] *\
+                                                                                                  fuel_shares["gas residential heat share"]*\
+                                                                                                  (1-fuel_shares["space to water heat share"])*\
+                                                                                                    base_energy_totals["residential gas"]*\
+                                                                                                     efficiency_heat_oil_to_elec)
+
+    energy_totals["residential heat oil"] =base_energy_totals["residential oil"] * fuel_shares["oil residential heat share"]* (1-fuel_shares["oil to elec heat share"])*\
+                                                    efficiency_gains["residential heat oil"] * growth_factors["residential heat oil"]
+    
+    energy_totals["residential oil"] =  base_energy_totals["residential oil"] * (1-fuel_shares["oil residential heat share"])* (1-fuel_shares["oil to elec share"])*\
+                                                    efficiency_gains["residential oil"] * growth_factors["residential oil"]
+
+    energy_totals["residential heat biomass"] = base_energy_totals["residential biomass"] * fuel_shares["biomass residential heat share"] *(1-fuel_shares["biomass to elec heat share"])*\
+                                                    efficiency_gains["residential heat biomass"] * growth_factors["residential heat biomass"]
+    
+    energy_totals["residential biomass"] = base_energy_totals["residential biomass"] * (1-fuel_shares["biomass residential heat share"]) *(1-fuel_shares["biomass to elec share"])*\
+                                                    efficiency_gains["residential biomass"] * growth_factors["residential biomass"]
+
+    energy_totals["residential heat gas"] =base_energy_totals["residential gas"] * fuel_shares["gas residential heat share"]* (1-fuel_shares["gas to elec heat share"])*\
+                                                    efficiency_gains["residential heat gas"] * growth_factors["residential heat gas"]
+    
+    energy_totals["residential gas"] =  base_energy_totals["residential gas"] * (1-fuel_shares["gas residential heat share"])* (1-fuel_shares["gas to elec share"])*\
+                                                    efficiency_gains["residential gas"] * growth_factors["residential gas"]
+
+    energy_totals["total residential space"] = energy_totals["electricity residential space"] + \
+                    (energy_totals["residential heat oil"] + energy_totals["residential heat biomass"]  +energy_totals["residential heat gas"]) * (fuel_shares["space to water heat share"])
+    
+    energy_totals["total residential water"] = energy_totals["electricity residential water"] + \
+                    (energy_totals["residential heat oil"] + energy_totals["residential heat biomass"]  +energy_totals["residential heat gas"]) *  (1 - fuel_shares["space to water heat share"])
+
+    energy_totals["electricity residential"] = energy_totals["electricity residential"] + \
+                (fuel_shares["oil to elec share"] * (1-fuel_shares["oil residential heat share"]) * base_energy_totals["residential oil"]) +\
+                (fuel_shares["biomass to elec share"] * (1-fuel_shares["biomass residential heat share"])* base_energy_totals["residential biomass"])+\
+                (fuel_shares["gas to elec share"] * (1-fuel_shares["gas residential heat share"]) * base_energy_totals["residential gas"])
+
+
+
+    # Road
     energy_totals["total road"] = (
-        (1 - fuel_cell_share + electric_share)
+        (1 - fuel_cell_share - electric_share)
         * efficiency_gains["total road ice"]
         * base_energy_totals["total road"]
         + fuel_cell_share
@@ -72,9 +145,10 @@ def get(item, investment_year=None):
         + electric_share
         * efficiency_gains["total road ev"]
         * base_energy_totals["total road"]
-    )
-
-    energy_totals["total domestic navigation"] = (
+    ) * growth_factors["total road"]
+    
+    #Navigation
+    energy_totals["total domestic navigation"] = ((
         1 - hydrogen_shipping_share
     ) * efficiency_gains["total navigation oil"] * base_energy_totals[
         "total domestic navigation"
@@ -82,9 +156,9 @@ def get(item, investment_year=None):
         "total navigation hydrogen"
     ] * base_energy_totals[
         "total domestic navigation"
-    ]
+    ]) * growth_factors["total domestic navigation"]
 
-    energy_totals["total international navigation"] = (
+    energy_totals["total international navigation"] = ((
         1 - hydrogen_shipping_share
     ) * efficiency_gains["total navigation oil"] * base_energy_totals[
         "total international navigation"
@@ -92,11 +166,10 @@ def get(item, investment_year=None):
         "total navigation hydrogen"
     ] * base_energy_totals[
         "total international navigation"
-    ]
+    ]) * growth_factors["total international navigation"]
+    
+    energy_totals["district heat share"] = district_heating["current"]
 
-    energy_totals["district heat share"] = 0
-    energy_totals["electricity residential water"] = 0
-    energy_totals["electricity residential space"] = 0
     energy_totals["electricity services space"] = 0
     energy_totals["electricity services water"] = 0
 

From 1aa33e63a29a308f62b999ee77cc4d28389a4436 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 23 Aug 2023 18:16:26 +0000
Subject: [PATCH 20/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/build_base_energy_totals.py |   4 +-
 scripts/prepare_energy_totals.py    | 241 ++++++++++++++++++----------
 2 files changed, 157 insertions(+), 88 deletions(-)

diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index 0ac7e30d..6d9548fa 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -173,8 +173,8 @@ def calc_sector(sector):
                 energy_totals_base.at[country, "total rail"] = (
                     df_sector[
                         (df_sector.Commodity == "Gas Oil/ Diesel Oil")
-                        | (df_sector.Commodity == "Biodiesel") 
-                            | (df_sector.Commodity == "Electricity")
+                        | (df_sector.Commodity == "Biodiesel")
+                        | (df_sector.Commodity == "Electricity")
                     ]
                     .Quantity_TWh.sum()
                     .round(4)
diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index c4db7610..d9f4b104 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -64,75 +64,146 @@ def get(item, investment_year=None):
 
     energy_totals = base_energy_totals * efficiency_gains * growth_factors
 
-
-    #Residential
+    # Residential
     efficiency_heat_oil_to_elec = 0.9
     efficiency_heat_biomass_to_elec = 0.9
 
-    energy_totals["electricity residential space"] = base_energy_totals["total residential space"] + (fuel_shares["biomass to elec heat share"] *\
-                                                                                                  fuel_shares["biomass residential heat share"]*\
-                                                                                                  (fuel_shares["space to water heat share"])*\
-                                                                                                    base_energy_totals["residential biomass"]*\
-                                                                                                     efficiency_heat_biomass_to_elec)\
-                                                                                                    \
-                                                                                            +  (fuel_shares["oil to elec heat share"] *\
-                                                                                                  fuel_shares["oil residential heat share"]*\
-                                                                                                  (fuel_shares["space to water heat share"])*\
-                                                                                                    base_energy_totals["residential oil"]*\
-                                                                                                     efficiency_heat_oil_to_elec)\
-                                                                                             +  (fuel_shares["gas to elec heat share"] *\
-                                                                                                  fuel_shares["gas residential heat share"]*\
-                                                                                                  (fuel_shares["space to water heat share"])*\
-                                                                                                    base_energy_totals["residential gas"]*\
-                                                                                                     efficiency_heat_oil_to_elec)
-
-    energy_totals["electricity residential water"] = base_energy_totals["total residential water"] + (fuel_shares["biomass to elec heat share"] *\
-                                                                                                  fuel_shares["biomass residential heat share"]*
-                                                                                                  (1-fuel_shares["space to water heat share"])*\
-                                                                                                    base_energy_totals["residential biomass"]*\
-                                                                                                     efficiency_heat_biomass_to_elec)\
-                                                                                                    \
-                                                                                            +  (fuel_shares["oil to elec heat share"] *\
-                                                                                                  fuel_shares["oil residential heat share"]*\
-                                                                                                  (1-fuel_shares["space to water heat share"])*\
-                                                                                                    base_energy_totals["residential oil"]*\
-                                                                                                     efficiency_heat_oil_to_elec)\
-                                                                                                +  (fuel_shares["gas to elec heat share"] *\
-                                                                                                  fuel_shares["gas residential heat share"]*\
-                                                                                                  (1-fuel_shares["space to water heat share"])*\
-                                                                                                    base_energy_totals["residential gas"]*\
-                                                                                                     efficiency_heat_oil_to_elec)
-
-    energy_totals["residential heat oil"] =base_energy_totals["residential oil"] * fuel_shares["oil residential heat share"]* (1-fuel_shares["oil to elec heat share"])*\
-                                                    efficiency_gains["residential heat oil"] * growth_factors["residential heat oil"]
-    
-    energy_totals["residential oil"] =  base_energy_totals["residential oil"] * (1-fuel_shares["oil residential heat share"])* (1-fuel_shares["oil to elec share"])*\
-                                                    efficiency_gains["residential oil"] * growth_factors["residential oil"]
-
-    energy_totals["residential heat biomass"] = base_energy_totals["residential biomass"] * fuel_shares["biomass residential heat share"] *(1-fuel_shares["biomass to elec heat share"])*\
-                                                    efficiency_gains["residential heat biomass"] * growth_factors["residential heat biomass"]
-    
-    energy_totals["residential biomass"] = base_energy_totals["residential biomass"] * (1-fuel_shares["biomass residential heat share"]) *(1-fuel_shares["biomass to elec share"])*\
-                                                    efficiency_gains["residential biomass"] * growth_factors["residential biomass"]
-
-    energy_totals["residential heat gas"] =base_energy_totals["residential gas"] * fuel_shares["gas residential heat share"]* (1-fuel_shares["gas to elec heat share"])*\
-                                                    efficiency_gains["residential heat gas"] * growth_factors["residential heat gas"]
-    
-    energy_totals["residential gas"] =  base_energy_totals["residential gas"] * (1-fuel_shares["gas residential heat share"])* (1-fuel_shares["gas to elec share"])*\
-                                                    efficiency_gains["residential gas"] * growth_factors["residential gas"]
-
-    energy_totals["total residential space"] = energy_totals["electricity residential space"] + \
-                    (energy_totals["residential heat oil"] + energy_totals["residential heat biomass"]  +energy_totals["residential heat gas"]) * (fuel_shares["space to water heat share"])
-    
-    energy_totals["total residential water"] = energy_totals["electricity residential water"] + \
-                    (energy_totals["residential heat oil"] + energy_totals["residential heat biomass"]  +energy_totals["residential heat gas"]) *  (1 - fuel_shares["space to water heat share"])
-
-    energy_totals["electricity residential"] = energy_totals["electricity residential"] + \
-                (fuel_shares["oil to elec share"] * (1-fuel_shares["oil residential heat share"]) * base_energy_totals["residential oil"]) +\
-                (fuel_shares["biomass to elec share"] * (1-fuel_shares["biomass residential heat share"])* base_energy_totals["residential biomass"])+\
-                (fuel_shares["gas to elec share"] * (1-fuel_shares["gas residential heat share"]) * base_energy_totals["residential gas"])
+    energy_totals["electricity residential space"] = (
+        base_energy_totals["total residential space"]
+        + (
+            fuel_shares["biomass to elec heat share"]
+            * fuel_shares["biomass residential heat share"]
+            * (fuel_shares["space to water heat share"])
+            * base_energy_totals["residential biomass"]
+            * efficiency_heat_biomass_to_elec
+        )
+        + (
+            fuel_shares["oil to elec heat share"]
+            * fuel_shares["oil residential heat share"]
+            * (fuel_shares["space to water heat share"])
+            * base_energy_totals["residential oil"]
+            * efficiency_heat_oil_to_elec
+        )
+        + (
+            fuel_shares["gas to elec heat share"]
+            * fuel_shares["gas residential heat share"]
+            * (fuel_shares["space to water heat share"])
+            * base_energy_totals["residential gas"]
+            * efficiency_heat_oil_to_elec
+        )
+    )
+
+    energy_totals["electricity residential water"] = (
+        base_energy_totals["total residential water"]
+        + (
+            fuel_shares["biomass to elec heat share"]
+            * fuel_shares["biomass residential heat share"]
+            * (1 - fuel_shares["space to water heat share"])
+            * base_energy_totals["residential biomass"]
+            * efficiency_heat_biomass_to_elec
+        )
+        + (
+            fuel_shares["oil to elec heat share"]
+            * fuel_shares["oil residential heat share"]
+            * (1 - fuel_shares["space to water heat share"])
+            * base_energy_totals["residential oil"]
+            * efficiency_heat_oil_to_elec
+        )
+        + (
+            fuel_shares["gas to elec heat share"]
+            * fuel_shares["gas residential heat share"]
+            * (1 - fuel_shares["space to water heat share"])
+            * base_energy_totals["residential gas"]
+            * efficiency_heat_oil_to_elec
+        )
+    )
+
+    energy_totals["residential heat oil"] = (
+        base_energy_totals["residential oil"]
+        * fuel_shares["oil residential heat share"]
+        * (1 - fuel_shares["oil to elec heat share"])
+        * efficiency_gains["residential heat oil"]
+        * growth_factors["residential heat oil"]
+    )
+
+    energy_totals["residential oil"] = (
+        base_energy_totals["residential oil"]
+        * (1 - fuel_shares["oil residential heat share"])
+        * (1 - fuel_shares["oil to elec share"])
+        * efficiency_gains["residential oil"]
+        * growth_factors["residential oil"]
+    )
 
+    energy_totals["residential heat biomass"] = (
+        base_energy_totals["residential biomass"]
+        * fuel_shares["biomass residential heat share"]
+        * (1 - fuel_shares["biomass to elec heat share"])
+        * efficiency_gains["residential heat biomass"]
+        * growth_factors["residential heat biomass"]
+    )
 
+    energy_totals["residential biomass"] = (
+        base_energy_totals["residential biomass"]
+        * (1 - fuel_shares["biomass residential heat share"])
+        * (1 - fuel_shares["biomass to elec share"])
+        * efficiency_gains["residential biomass"]
+        * growth_factors["residential biomass"]
+    )
+
+    energy_totals["residential heat gas"] = (
+        base_energy_totals["residential gas"]
+        * fuel_shares["gas residential heat share"]
+        * (1 - fuel_shares["gas to elec heat share"])
+        * efficiency_gains["residential heat gas"]
+        * growth_factors["residential heat gas"]
+    )
+
+    energy_totals["residential gas"] = (
+        base_energy_totals["residential gas"]
+        * (1 - fuel_shares["gas residential heat share"])
+        * (1 - fuel_shares["gas to elec share"])
+        * efficiency_gains["residential gas"]
+        * growth_factors["residential gas"]
+    )
+
+    energy_totals["total residential space"] = energy_totals[
+        "electricity residential space"
+    ] + (
+        energy_totals["residential heat oil"]
+        + energy_totals["residential heat biomass"]
+        + energy_totals["residential heat gas"]
+    ) * (
+        fuel_shares["space to water heat share"]
+    )
+
+    energy_totals["total residential water"] = energy_totals[
+        "electricity residential water"
+    ] + (
+        energy_totals["residential heat oil"]
+        + energy_totals["residential heat biomass"]
+        + energy_totals["residential heat gas"]
+    ) * (
+        1 - fuel_shares["space to water heat share"]
+    )
+
+    energy_totals["electricity residential"] = (
+        energy_totals["electricity residential"]
+        + (
+            fuel_shares["oil to elec share"]
+            * (1 - fuel_shares["oil residential heat share"])
+            * base_energy_totals["residential oil"]
+        )
+        + (
+            fuel_shares["biomass to elec share"]
+            * (1 - fuel_shares["biomass residential heat share"])
+            * base_energy_totals["residential biomass"]
+        )
+        + (
+            fuel_shares["gas to elec share"]
+            * (1 - fuel_shares["gas residential heat share"])
+            * base_energy_totals["residential gas"]
+        )
+    )
 
     # Road
     energy_totals["total road"] = (
@@ -146,28 +217,26 @@ def get(item, investment_year=None):
         * efficiency_gains["total road ev"]
         * base_energy_totals["total road"]
     ) * growth_factors["total road"]
-    
-    #Navigation
-    energy_totals["total domestic navigation"] = ((
-        1 - hydrogen_shipping_share
-    ) * efficiency_gains["total navigation oil"] * base_energy_totals[
-        "total domestic navigation"
-    ] + hydrogen_shipping_share * efficiency_gains[
-        "total navigation hydrogen"
-    ] * base_energy_totals[
-        "total domestic navigation"
-    ]) * growth_factors["total domestic navigation"]
-
-    energy_totals["total international navigation"] = ((
-        1 - hydrogen_shipping_share
-    ) * efficiency_gains["total navigation oil"] * base_energy_totals[
-        "total international navigation"
-    ] + hydrogen_shipping_share * efficiency_gains[
-        "total navigation hydrogen"
-    ] * base_energy_totals[
-        "total international navigation"
-    ]) * growth_factors["total international navigation"]
-    
+
+    # Navigation
+    energy_totals["total domestic navigation"] = (
+        (1 - hydrogen_shipping_share)
+        * efficiency_gains["total navigation oil"]
+        * base_energy_totals["total domestic navigation"]
+        + hydrogen_shipping_share
+        * efficiency_gains["total navigation hydrogen"]
+        * base_energy_totals["total domestic navigation"]
+    ) * growth_factors["total domestic navigation"]
+
+    energy_totals["total international navigation"] = (
+        (1 - hydrogen_shipping_share)
+        * efficiency_gains["total navigation oil"]
+        * base_energy_totals["total international navigation"]
+        + hydrogen_shipping_share
+        * efficiency_gains["total navigation hydrogen"]
+        * base_energy_totals["total international navigation"]
+    ) * growth_factors["total international navigation"]
+
     energy_totals["district heat share"] = district_heating["current"]
 
     energy_totals["electricity services space"] = 0

From 78154fd50525885942df5602069874d48658125a Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 23 Aug 2023 20:18:29 +0200
Subject: [PATCH 21/58] adapt csv files needed for demand workflow

---
 data/demand/district_heating.csv | 4 ++++
 data/demand/efficiency_gains.csv | 4 ++--
 data/demand/fuel_shares.csv      | 2 ++
 data/demand/growth_factors.csv   | 4 ++--
 4 files changed, 10 insertions(+), 4 deletions(-)
 create mode 100644 data/demand/district_heating.csv
 create mode 100644 data/demand/fuel_shares.csv

diff --git a/data/demand/district_heating.csv b/data/demand/district_heating.csv
new file mode 100644
index 00000000..388c8c30
--- /dev/null
+++ b/data/demand/district_heating.csv
@@ -0,0 +1,4 @@
+country,current,potential
+MA, 0, 0
+NG, 0, 0
+BJ, 0, 0
\ No newline at end of file
diff --git a/data/demand/efficiency_gains.csv b/data/demand/efficiency_gains.csv
index 557c51f4..ebaac4ce 100644
--- a/data/demand/efficiency_gains.csv
+++ b/data/demand/efficiency_gains.csv
@@ -1,2 +1,2 @@
-,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen
-MA,0.91,0.82,0.79,0.88,0.71,0.91,0.82,0.79,0.88,0.71,0.91,0.82,0.79,0.88,0.71,0.91,0.82,0.79,0.88,0.71,1.0, 0.9, 0.87, 0.99, 0.9, 0.85
+,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen,residential heat biomass,residential heat oil,residential heat gas
+MA,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,1.0,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85, 0.95, 0.95, 1.0
diff --git a/data/demand/fuel_shares.csv b/data/demand/fuel_shares.csv
new file mode 100644
index 00000000..b25865b8
--- /dev/null
+++ b/data/demand/fuel_shares.csv
@@ -0,0 +1,2 @@
+country,oil residential heat share,biomass residential heat share,biomass to elec heat share,oil to elec heat share,biomass to elec share,oil to elec share,space to water heat share,gas to elec share,gas to elec heat share,gas residential heat share
+MA, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
\ No newline at end of file
diff --git a/data/demand/growth_factors.csv b/data/demand/growth_factors.csv
index 3eca6c46..9f184f50 100644
--- a/data/demand/growth_factors.csv
+++ b/data/demand/growth_factors.csv
@@ -1,2 +1,2 @@
-,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas
-MA,1.5,1.6,1.3, 1.4,1.5,1.1,1.2,1.3,1.23,1.52,1.13,1.4,1.23,1.43,1.231,1.132,1.41,1.23,1.32,1.40,0.91,0.89
+,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,residential heat oil,residential heat biomass,residential heat gas
+MA,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.0,1.0,1.23,1.32,0.5,1.0,0.89, 1.0, 1.0, 1.0

From 6bad5bec15c30e99b566ab3c6190f5e52c57901e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 23 Aug 2023 18:18:49 +0000
Subject: [PATCH 22/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 data/demand/district_heating.csv | 2 +-
 data/demand/fuel_shares.csv      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/data/demand/district_heating.csv b/data/demand/district_heating.csv
index 388c8c30..122a5a1d 100644
--- a/data/demand/district_heating.csv
+++ b/data/demand/district_heating.csv
@@ -1,4 +1,4 @@
 country,current,potential
 MA, 0, 0
 NG, 0, 0
-BJ, 0, 0
\ No newline at end of file
+BJ, 0, 0
diff --git a/data/demand/fuel_shares.csv b/data/demand/fuel_shares.csv
index b25865b8..dfc5f9f3 100644
--- a/data/demand/fuel_shares.csv
+++ b/data/demand/fuel_shares.csv
@@ -1,2 +1,2 @@
 country,oil residential heat share,biomass residential heat share,biomass to elec heat share,oil to elec heat share,biomass to elec share,oil to elec share,space to water heat share,gas to elec share,gas to elec heat share,gas residential heat share
-MA, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
\ No newline at end of file
+MA, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75

From 9570636196dc874e1473c2f9b4f822064d681501 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 23 Aug 2023 20:28:01 +0200
Subject: [PATCH 23/58] add needed config params

---
 config.default.yaml              | 3 +++
 scripts/prepare_energy_totals.py | 9 +++++----
 test/config.test1.yaml           | 5 ++++-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index 6eedbfa6..07c1c58c 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -212,6 +212,9 @@ sector:
 
   biomass_transport: true  # biomass transport between nodes
 
+  efficiency_heat_oil_to_elec: 0.9
+  efficiency_heat_biomass_to_elec: 0.9
+  efficiency_heat_gas_to_elec: 0.9
 
   land_transport_fuel_cell_share: # 1 means all FCEVs HERE
     BU_2030: 0.00
diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index d9f4b104..d4d46d12 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -65,8 +65,9 @@ def get(item, investment_year=None):
     energy_totals = base_energy_totals * efficiency_gains * growth_factors
 
     # Residential
-    efficiency_heat_oil_to_elec = 0.9
-    efficiency_heat_biomass_to_elec = 0.9
+    efficiency_heat_oil_to_elec = snakemake.config["sector"]["efficiency_heat_oil_to_elec"]
+    efficiency_heat_biomass_to_elec = snakemake.config["sector"]["efficiency_heat_biomass_to_elec"]
+    efficiency_heat_gas_to_elec = snakemake.config["sector"]["efficiency_heat_gas_to_elec"]
 
     energy_totals["electricity residential space"] = (
         base_energy_totals["total residential space"]
@@ -89,7 +90,7 @@ def get(item, investment_year=None):
             * fuel_shares["gas residential heat share"]
             * (fuel_shares["space to water heat share"])
             * base_energy_totals["residential gas"]
-            * efficiency_heat_oil_to_elec
+            * efficiency_heat_gas_to_elec
         )
     )
 
@@ -114,7 +115,7 @@ def get(item, investment_year=None):
             * fuel_shares["gas residential heat share"]
             * (1 - fuel_shares["space to water heat share"])
             * base_energy_totals["residential gas"]
-            * efficiency_heat_oil_to_elec
+            * efficiency_heat_gas_to_elec
         )
     )
 
diff --git a/test/config.test1.yaml b/test/config.test1.yaml
index ff323d3d..38d2fe8c 100644
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@@ -212,7 +212,10 @@ sector:
   transport_internal_combustion_efficiency: 0.3
 
   biomass_transport: true  # biomass transport between nodes
-
+  
+  efficiency_heat_oil_to_elec: 0.9
+  efficiency_heat_biomass_to_elec: 0.9
+  efficiency_heat_gas_to_elec: 0.9
 
   land_transport_fuel_cell_share: # 1 means all FCEVs HERE
     BU_2030: 0.00

From aefb6f8f397305b1fda160ca6685c743b35bf835 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Tue, 27 Sep 2022 13:01:22 +0200
Subject: [PATCH 24/58] adapt for elec residential heat demand

---
 data/demand/efficiency_gains copy.csv | 2 ++
 data/demand/growth_factors copy.csv   | 2 ++
 data/energy_totals_DF_2030.csv        | 9 ++-------
 data/energy_totals_XX_2030.csv        | 2 +-
 data/energy_totals_XX_2030_.csv       | 2 ++
 data/energy_totals_XX_2038.csv        | 2 ++
 6 files changed, 11 insertions(+), 8 deletions(-)
 create mode 100644 data/demand/efficiency_gains copy.csv
 create mode 100644 data/demand/growth_factors copy.csv
 create mode 100644 data/energy_totals_XX_2030_.csv
 create mode 100644 data/energy_totals_XX_2038.csv

diff --git a/data/demand/efficiency_gains copy.csv b/data/demand/efficiency_gains copy.csv
new file mode 100644
index 00000000..43fc4975
--- /dev/null
+++ b/data/demand/efficiency_gains copy.csv	
@@ -0,0 +1,2 @@
+,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen
+MA,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,0.95,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85
diff --git a/data/demand/growth_factors copy.csv b/data/demand/growth_factors copy.csv
new file mode 100644
index 00000000..8775d5f6
--- /dev/null
+++ b/data/demand/growth_factors copy.csv	
@@ -0,0 +1,2 @@
+,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas
+MA,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.132,1.41,1.23,1.32,0.5,1.0,0.89
diff --git a/data/energy_totals_DF_2030.csv b/data/energy_totals_DF_2030.csv
index d558dc9b..4ffa5e6b 100644
--- a/data/energy_totals_DF_2030.csv
+++ b/data/energy_totals_DF_2030.csv
@@ -1,7 +1,2 @@
-country,total residential space,electricity residential space,total residential water,electricity residential water,total residential cooking,electricity residential cooking,total residential,electricity residential,derived heat residential,thermal uses residential,total services space,electricity services space,total services water,electricity services water,total services cooking,electricity services cooking,total services,electricity services,derived heat services,thermal uses services,total agriculture electricity,total agriculture heat,total agriculture machinery,total agriculture,total road,electricity road,total two-wheel,total passenger cars,electricity passenger cars,total other road passenger,electricity other road passenger,total light duty road freight,electricity light duty road freight,total heavy duty road freight,total rail,electricity rail,total rail passenger,electricity rail passenger,total rail freight,electricity rail freight,total aviation passenger,total aviation freight,total domestic aviation passenger,total international aviation passenger,total domestic aviation freight,total international aviation freight,total domestic aviation,total international aviation,total domestic navigation,district heat share,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass
-MA,4.2,0.28,7.3,0.23,4.9,3.3,23.31,8.03,,,,,,,,,15.89,9.13,,,4.91,0,,12.03,64.23,,,,,,,,,,,,,,,,,,,,,,0.39,8.93,10.32,,1.32,3.321242,3.66435234,5.344235432,3.6,2.7
-NG,4.2,0.28,7.3,0.23,4.9,3.3,23.31,8.03,,,,,,,,,15.89,9.13,,,4.91,0,,12.03,64.23,,,,,,,,,,,,,,,,,,,,,,0.39,8.93,10.32,,1.32,3.321242,3.66435234,5.344235432,3.6,2.7
-BJ,4.2,0.28,7.3,0.23,4.9,3.3,23.31,8.03,,,,,,,,,15.89,9.13,,,4.91,0,,12.03,64.23,,,,,,,,,,,,,,,,,,,,,,0.39,8.93,10.32,,1.32,3.321242,3.66435234,5.344235432,3.6,2.7
-NA,4.2,0.28,7.3,0.23,4.9,3.3,23.31,8.03,,,,,,,,,15.89,9.13,,,4.91,0,,12.03,64.23,,,,,,,,,,,,,,,,,,,,,,0.39,8.93,10.32,,1.32,3.321242,3.66435234,5.344235432,3.6,2.7
-BR,4.2,0.28,7.3,0.23,4.9,3.3,23.31,8.03,,,,,,,,,15.89,9.13,,,4.91,0,,12.03,64.23,,,,,,,,,,,,,,,,,,,,,,0.39,8.93,10.32,,1.32,3.321242,3.66435234,5.344235432,3.6,2.7
-AE,4.2,0.28,7.3,0.23,4.9,3.3,23.31,8.03,,,,,,,,,15.89,9.13,,,4.91,0,,12.03,64.23,,,,,,,,,,,,,,,,,,,,,,0.39,8.93,10.32,,1.32,3.321242,3.66435234,5.344235432,3.6,2.7
+,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
+MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.693556480000002,1.9635325499999998,0.1264536,0.0,0.0,85.52591748000002,0.0,0.0,0,0,0,0,0
diff --git a/data/energy_totals_XX_2030.csv b/data/energy_totals_XX_2030.csv
index 4e454a61..807e8a0b 100644
--- a/data/energy_totals_XX_2030.csv
+++ b/data/energy_totals_XX_2030.csv
@@ -1,2 +1,2 @@
 ,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
-MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.693556480000002,1.40252325,0.1264536,0.0,0.0,71.27159790000002,0.0,0.0,0,0,0,0,0
+MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,33.67658116,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.693556480000002,1.9635325499999998,0.1264536,0.0,0.0,85.63625160000001,0.0,0.0,0,0,0,0,0
diff --git a/data/energy_totals_XX_2030_.csv b/data/energy_totals_XX_2030_.csv
new file mode 100644
index 00000000..4e454a61
--- /dev/null
+++ b/data/energy_totals_XX_2030_.csv
@@ -0,0 +1,2 @@
+,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
+MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.693556480000002,1.40252325,0.1264536,0.0,0.0,71.27159790000002,0.0,0.0,0,0,0,0,0
diff --git a/data/energy_totals_XX_2038.csv b/data/energy_totals_XX_2038.csv
new file mode 100644
index 00000000..b9645598
--- /dev/null
+++ b/data/energy_totals_XX_2038.csv
@@ -0,0 +1,2 @@
+,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential heat biomass,residential heat gas,residential heat oil,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,electricity residential space,electricity residential water,district heat share,electricity services space,electricity services water
+MA,0.0,4.7861475,6.9489728,0.7777727999999999,35.97336541,1.250795,0.0,3.917325,0.0,9.917039160499998,5.218711410000001,6.91083,11.17675,0.0,0.31075,0.792,0.0,14.2347,2.40155736,1.1411400000000003,13.937672334899998,9.2917815566,73.2802447,0.0,0.0,5.6370538386,3.7580358924,0,0,0

From 211ca4fabca57589af2e13fcdd445519a215b023 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 23 Aug 2023 18:39:39 +0000
Subject: [PATCH 25/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_energy_totals.py | 12 +++++++++---
 test/config.test1.yaml           |  1 -
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index d4d46d12..7d39e078 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -65,9 +65,15 @@ def get(item, investment_year=None):
     energy_totals = base_energy_totals * efficiency_gains * growth_factors
 
     # Residential
-    efficiency_heat_oil_to_elec = snakemake.config["sector"]["efficiency_heat_oil_to_elec"]
-    efficiency_heat_biomass_to_elec = snakemake.config["sector"]["efficiency_heat_biomass_to_elec"]
-    efficiency_heat_gas_to_elec = snakemake.config["sector"]["efficiency_heat_gas_to_elec"]
+    efficiency_heat_oil_to_elec = snakemake.config["sector"][
+        "efficiency_heat_oil_to_elec"
+    ]
+    efficiency_heat_biomass_to_elec = snakemake.config["sector"][
+        "efficiency_heat_biomass_to_elec"
+    ]
+    efficiency_heat_gas_to_elec = snakemake.config["sector"][
+        "efficiency_heat_gas_to_elec"
+    ]
 
     energy_totals["electricity residential space"] = (
         base_energy_totals["total residential space"]
diff --git a/test/config.test1.yaml b/test/config.test1.yaml
index 38d2fe8c..972ebec2 100644
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@@ -212,7 +212,6 @@ sector:
   transport_internal_combustion_efficiency: 0.3
 
   biomass_transport: true  # biomass transport between nodes
-  
   efficiency_heat_oil_to_elec: 0.9
   efficiency_heat_biomass_to_elec: 0.9
   efficiency_heat_gas_to_elec: 0.9

From b5b65c47719381cb5a0eab541ffdb0a30fba4022 Mon Sep 17 00:00:00 2001
From: Hazem <87850910+hazemakhalek@users.noreply.github.com>
Date: Thu, 24 Aug 2023 11:15:27 +0200
Subject: [PATCH 26/58] Update build_base_energy_totals.py

---
 scripts/build_base_energy_totals.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index 6d9548fa..e070caf0 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -345,7 +345,7 @@ def calc_sector(sector):
 
     # Create an empty dataframe for energy_totals_base
     energy_totals_cols = pd.read_csv(
-        "/nfs/home/haz43975/pypsa-earth-sec/data/energy_totals_DF_2030.csv"
+        "data/energy_totals_DF_2030.csv"
     ).columns
     energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries)
 

From 41d9ce7e40bf898448a2c66aed75b56151120c95 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 24 Aug 2023 09:15:36 +0000
Subject: [PATCH 27/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/build_base_energy_totals.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index e070caf0..d281dcca 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -344,9 +344,7 @@ def calc_sector(sector):
     df_yr = df_yr[df_yr.country.isin(countries)]
 
     # Create an empty dataframe for energy_totals_base
-    energy_totals_cols = pd.read_csv(
-        "data/energy_totals_DF_2030.csv"
-    ).columns
+    energy_totals_cols = pd.read_csv("data/energy_totals_DF_2030.csv").columns
     energy_totals_base = pd.DataFrame(columns=energy_totals_cols, index=countries)
 
     # Lists that combine the different fuels in the dataset to the model's carriers

From cce5c48736287d7560986040f4f24ab63e1677a0 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 24 Aug 2023 11:55:09 +0200
Subject: [PATCH 28/58] add missing parameter in config.test

---
 test/config.test1.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/config.test1.yaml b/test/config.test1.yaml
index 3b00af20..a8f4d1c7 100644
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@@ -230,6 +230,7 @@ sector:
     AP_2030: 0.004
     NZ_2030: 0.02
     DF_2030: 0.01
+    AB_2030: 0.01
     BU_2050: 0.00
     AP_2050: 0.06
     NZ_2050: 0.28
@@ -240,6 +241,7 @@ sector:
     AP_2030: 0.075
     NZ_2030: 0.13
     DF_2030: 0.01
+    AB_2030: 0.01
     BU_2050: 0.00
     AP_2050: 0.42
     NZ_2050: 0.68
@@ -257,6 +259,7 @@ sector:
     AP_2030: 0.00
     NZ_2030: 0.10
     DF_2030: 0.05
+    AB_2030: 0.05
     BU_2050: 0.00
     AP_2050: 0.25
     NZ_2050: 0.36

From 54c68ee7986228569e3c481fef54ff81a7e1cff8 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 24 Aug 2023 13:35:09 +0200
Subject: [PATCH 29/58] adapting workflow for other countries

---
 data/demand/efficiency_gains.csv    |  2 ++
 data/demand/fuel_shares.csv         |  3 +++
 data/demand/growth_factors.csv      |  2 ++
 scripts/build_base_energy_totals.py |  1 +
 scripts/prepare_energy_totals.py    | 17 +++++++++++++----
 5 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/data/demand/efficiency_gains.csv b/data/demand/efficiency_gains.csv
index ebaac4ce..e55aa271 100644
--- a/data/demand/efficiency_gains.csv
+++ b/data/demand/efficiency_gains.csv
@@ -1,2 +1,4 @@
 ,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen,residential heat biomass,residential heat oil,residential heat gas
 MA,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,1.0,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85, 0.95, 0.95, 1.0
+NG,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,1.0,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85, 0.95, 0.95, 1.0
+BJ,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,1.0,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85, 0.95, 0.95, 1.0
diff --git a/data/demand/fuel_shares.csv b/data/demand/fuel_shares.csv
index dfc5f9f3..a3a4fa76 100644
--- a/data/demand/fuel_shares.csv
+++ b/data/demand/fuel_shares.csv
@@ -1,2 +1,5 @@
 country,oil residential heat share,biomass residential heat share,biomass to elec heat share,oil to elec heat share,biomass to elec share,oil to elec share,space to water heat share,gas to elec share,gas to elec heat share,gas residential heat share
 MA, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
+NG, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
+BJ, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
+
diff --git a/data/demand/growth_factors.csv b/data/demand/growth_factors.csv
index 9f184f50..6e09e3b9 100644
--- a/data/demand/growth_factors.csv
+++ b/data/demand/growth_factors.csv
@@ -1,2 +1,4 @@
 ,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,residential heat oil,residential heat biomass,residential heat gas
 MA,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.0,1.0,1.23,1.32,0.5,1.0,0.89, 1.0, 1.0, 1.0
+NG,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.0,1.0,1.23,1.32,0.5,1.0,0.89, 1.0, 1.0, 1.0
+BJ,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.0,1.0,1.23,1.32,0.5,1.0,0.89, 1.0, 1.0, 1.0
diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index d281dcca..a09b79d2 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -338,6 +338,7 @@ def calc_sector(sector):
     # Fetch country list and demand base year from the config file
     year = snakemake.config["demand_data"]["base_year"]
     countries = snakemake.config["countries"]
+    #countries = ["NG", "BJ"]
 
     # Filter for the year and country
     df_yr = df[df.Year == year]
diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index 7d39e078..ce9bf72e 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -34,21 +34,30 @@ def get(item, investment_year=None):
             "prepare_energy_totals",
             simpl="",
             clusters=10,
-            demand="XX",
-            planning_horizons=2038,
+            demand="AB",
+            planning_horizons=2030,
         )
         sets_path_to_root("pypsa-earth-sec")
 
+    countries = snakemake.config["countries"]
+    #countries = ["NG", "BJ"]
+    investment_year = int(snakemake.wildcards.planning_horizons)
+    demand_sc = snakemake.wildcards.demand  # loading the demand scenrario wildcard
+
     base_energy_totals = pd.read_csv("data/energy_totals_base.csv", index_col=0)
     growth_factors = pd.read_csv("data/demand/growth_factors.csv", index_col=0)
     efficiency_gains = pd.read_csv("data/demand/efficiency_gains.csv", index_col=0)
     fuel_shares = pd.read_csv("data/demand/fuel_shares.csv", index_col=0)
     district_heating = pd.read_csv("data/demand/district_heating.csv", index_col=0)
 
+    growth_factors = growth_factors[growth_factors.index.isin(countries)]
+    efficiency_gains = efficiency_gains[efficiency_gains.index.isin(countries)]
+    fuel_shares = fuel_shares[fuel_shares.index.isin(countries)]
+    district_heating = district_heating[district_heating.index.isin(countries)]
+
     options = snakemake.config["sector"]
 
-    investment_year = int(snakemake.wildcards.planning_horizons)
-    demand_sc = snakemake.wildcards.demand  # loading the demand scenrario wildcard
+
 
     fuel_cell_share = get(
         options["land_transport_fuel_cell_share"],

From cc597167562c95ca41f6ce910513a403e8041dde Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 24 Aug 2023 11:35:25 +0000
Subject: [PATCH 30/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 data/demand/fuel_shares.csv         | 1 -
 scripts/build_base_energy_totals.py | 2 +-
 scripts/prepare_energy_totals.py    | 4 +---
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/data/demand/fuel_shares.csv b/data/demand/fuel_shares.csv
index a3a4fa76..a380b12e 100644
--- a/data/demand/fuel_shares.csv
+++ b/data/demand/fuel_shares.csv
@@ -2,4 +2,3 @@ country,oil residential heat share,biomass residential heat share,biomass to ele
 MA, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
 NG, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
 BJ, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
-
diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index a09b79d2..35794ee8 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -338,7 +338,7 @@ def calc_sector(sector):
     # Fetch country list and demand base year from the config file
     year = snakemake.config["demand_data"]["base_year"]
     countries = snakemake.config["countries"]
-    #countries = ["NG", "BJ"]
+    # countries = ["NG", "BJ"]
 
     # Filter for the year and country
     df_yr = df[df.Year == year]
diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index ce9bf72e..d820b11b 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -40,7 +40,7 @@ def get(item, investment_year=None):
         sets_path_to_root("pypsa-earth-sec")
 
     countries = snakemake.config["countries"]
-    #countries = ["NG", "BJ"]
+    # countries = ["NG", "BJ"]
     investment_year = int(snakemake.wildcards.planning_horizons)
     demand_sc = snakemake.wildcards.demand  # loading the demand scenrario wildcard
 
@@ -57,8 +57,6 @@ def get(item, investment_year=None):
 
     options = snakemake.config["sector"]
 
-
-
     fuel_cell_share = get(
         options["land_transport_fuel_cell_share"],
         demand_sc + "_" + str(investment_year),

From b16ba497835fff513a8a4f483ea00313f74eb932 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 24 Aug 2023 14:18:54 +0200
Subject: [PATCH 31/58] make residential with all carreirs a default sector

---
 scripts/prepare_energy_totals.py  |   8 +-
 scripts/prepare_sector_network.py | 316 +++++++++++++++---------------
 scripts/prepare_transport_data.py |   2 +-
 3 files changed, 163 insertions(+), 163 deletions(-)

diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index d820b11b..e3faa738 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -50,10 +50,10 @@ def get(item, investment_year=None):
     fuel_shares = pd.read_csv("data/demand/fuel_shares.csv", index_col=0)
     district_heating = pd.read_csv("data/demand/district_heating.csv", index_col=0)
 
-    growth_factors = growth_factors[growth_factors.index.isin(countries)]
-    efficiency_gains = efficiency_gains[efficiency_gains.index.isin(countries)]
-    fuel_shares = fuel_shares[fuel_shares.index.isin(countries)]
-    district_heating = district_heating[district_heating.index.isin(countries)]
+    # growth_factors = growth_factors[growth_factors.index.isin(countries)]
+    # efficiency_gains = efficiency_gains[efficiency_gains.index.isin(countries)]
+    # fuel_shares = fuel_shares[fuel_shares.index.isin(countries)]
+    # district_heating = district_heating[district_heating.index.isin(countries)]
 
     options = snakemake.config["sector"]
 
diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index ce8ea030..06b5f2c0 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -2058,184 +2058,184 @@ def add_agriculture(n, costs):
 def add_residential(n, costs):
     # need to adapt for many countries #TODO
 
-    if snakemake.config["custom_data"]["heat_demand"]:
-        # heat_demand_index=n.loads_t.p.filter(like='residential').filter(like='heat').dropna(axis=1).index
-        # oil_res_index=n.loads_t.p.filter(like='residential').filter(like='oil').dropna(axis=1).index
-
-        heat_ind = (
-            n.loads_t.p_set.filter(like=countries[0])
-            .filter(like="residential")
-            .filter(like="heat")
-            .dropna(axis=1)
-            .columns
-        )
-        oil_ind = (
-            n.loads_t.p_set.filter(like=countries[0])
-            .filter(like="residential")
-            .filter(like="oil")
-            .dropna(axis=1)
-            .columns
-        )
-        bio_ind = (
-            n.loads_t.p_set.filter(like=countries[0])
-            .filter(like="residential")
-            .filter(like="biomass")
-            .dropna(axis=1)
-            .columns
-        )
+    # if snakemake.config["custom_data"]["heat_demand"]:
+    # heat_demand_index=n.loads_t.p.filter(like='residential').filter(like='heat').dropna(axis=1).index
+    # oil_res_index=n.loads_t.p.filter(like='residential').filter(like='oil').dropna(axis=1).index
+
+    heat_ind = (
+        n.loads_t.p_set.filter(like=countries[0])
+        .filter(like="residential")
+        .filter(like="heat")
+        .dropna(axis=1)
+        .columns
+    )
+    oil_ind = (
+        n.loads_t.p_set.filter(like=countries[0])
+        .filter(like="residential")
+        .filter(like="oil")
+        .dropna(axis=1)
+        .columns
+    )
+    bio_ind = (
+        n.loads_t.p_set.filter(like=countries[0])
+        .filter(like="residential")
+        .filter(like="biomass")
+        .dropna(axis=1)
+        .columns
+    )
+
+    gas_ind = (
+        n.loads_t.p_set.filter(like=countries[0])
+        .filter(like="residential")
+        .filter(like="gas")
+        .dropna(axis=1)
+        .columns
+    )
+
+    heat_shape = (
+        n.loads_t.p_set.loc[:, heat_ind]
+        / n.loads_t.p_set.loc[:, heat_ind].sum().sum()
+    )
+    heat_oil_demand = (
+        heat_shape * energy_totals.loc[countries[0], "residential heat oil"] * 1e6
+    )
+
+    heat_biomass_demand = (
+        heat_shape
+        * energy_totals.loc[countries[0], "residential heat biomass"]
+        * 1e6
+    )
 
-        gas_ind = (
-            n.loads_t.p_set.filter(like=countries[0])
-            .filter(like="residential")
-            .filter(like="gas")
-            .dropna(axis=1)
-            .columns
-        )
+    heat_gas_demand = (
+        heat_shape * energy_totals.loc[countries[0], "residential heat gas"] * 1e6
+    )
 
-        heat_shape = (
-            n.loads_t.p_set.loc[:, heat_ind]
-            / n.loads_t.p_set.loc[:, heat_ind].sum().sum()
-        )
-        heat_oil_demand = (
-            heat_shape * energy_totals.loc[countries[0], "residential heat oil"] * 1e6
+    n.loads_t.p_set.loc[:, heat_ind] = (
+        heat_shape
+        * (
+            energy_totals.loc[countries, "total residential space"].sum()
+            + energy_totals.loc[countries, "total residential water"].sum()
+            - energy_totals.loc[countries[0], "residential heat biomass"]
+            - energy_totals.loc[countries[0], "residential heat oil"]
+            - energy_totals.loc[countries[0], "residential heat gas"]
         )
+        * 1e6
+    )
 
-        heat_biomass_demand = (
-            heat_shape
-            * energy_totals.loc[countries[0], "residential heat biomass"]
-            * 1e6
-        )
+    # TODO make compatible with more counties
+    profile_residential = (
+        n.loads_t.p_set[nodes] / n.loads_t.p_set[nodes].sum().sum()
+    )
 
-        heat_gas_demand = (
-            heat_shape * energy_totals.loc[countries[0], "residential heat gas"] * 1e6
-        )
+    p_set_oil = (
+        profile_residential
+        * energy_totals.loc[countries, "residential oil"].sum()
+        * 1e6
+    ) + heat_oil_demand.values
 
-        n.loads_t.p_set.loc[:, heat_ind] = (
-            heat_shape
-            * (
-                energy_totals.loc[countries, "total residential space"].sum()
-                + energy_totals.loc[countries, "total residential water"].sum()
-                - energy_totals.loc[countries[0], "residential heat biomass"]
-                - energy_totals.loc[countries[0], "residential heat oil"]
-                - energy_totals.loc[countries[0], "residential heat gas"]
-            )
-            * 1e6
-        )
+    p_set_biomass = (
+        profile_residential
+        * energy_totals.loc[countries, "residential biomass"].sum()
+        * 1e6
+    ) + heat_biomass_demand.values
 
-        # TODO make compatible with more counties
-        profile_residential = (
-            n.loads_t.p_set[nodes] / n.loads_t.p_set[nodes].sum().sum()
-        )
+    p_set_gas = (
+        profile_residential
+        * energy_totals.loc[countries, "residential gas"].sum()
+        * 1e6
+    ) + heat_gas_demand.values
 
-        p_set_oil = (
-            profile_residential
-            * energy_totals.loc[countries, "residential oil"].sum()
-            * 1e6
-        ) + heat_oil_demand.values
+    n.madd(
+        "Load",
+        nodes,
+        suffix=" residential oil",
+        bus=spatial.oil.nodes,
+        carrier="residential oil",
+        p_set=p_set_oil,
+    )
+    co2 = (p_set_oil.sum().sum() * costs.at["oil", "CO2 intensity"]) / 8760
 
-        p_set_biomass = (
-            profile_residential
-            * energy_totals.loc[countries, "residential biomass"].sum()
-            * 1e6
-        ) + heat_biomass_demand.values
+    n.add(
+        "Load",
+        "residential oil emissions",
+        bus="co2 atmosphere",
+        carrier="oil emissions",
+        p_set=-co2,
+    )
+    n.madd(
+        "Load",
+        nodes,
+        suffix=" residential biomass",
+        bus=spatial.biomass.nodes,
+        carrier="residential biomass",
+        p_set=p_set_biomass,
+    )
 
-        p_set_gas = (
-            profile_residential
-            * energy_totals.loc[countries, "residential gas"].sum()
-            * 1e6
-        ) + heat_gas_demand.values
+    n.madd(
+        "Load",
+        nodes,
+        suffix=" residential gas",
+        bus=spatial.gas.nodes,
+        carrier="residential gas",
+        p_set=p_set_gas,
+    )
+    co2 = (p_set_gas.sum().sum() * costs.at["gas", "CO2 intensity"]) / 8760
 
-        n.madd(
-            "Load",
-            nodes,
-            suffix=" residential oil",
-            bus=spatial.oil.nodes,
-            carrier="residential oil",
-            p_set=p_set_oil,
-        )
-        co2 = (p_set_oil.sum().sum() * costs.at["oil", "CO2 intensity"]) / 8760
+    n.add(
+        "Load",
+        "residential gas emissions",
+        bus="co2 atmosphere",
+        carrier="gas emissions",
+        p_set=-co2,
+    )
 
-        n.add(
-            "Load",
-            "residential oil emissions",
-            bus="co2 atmosphere",
-            carrier="oil emissions",
-            p_set=-co2,
-        )
-        n.madd(
-            "Load",
-            nodes,
-            suffix=" residential biomass",
-            bus=spatial.biomass.nodes,
-            carrier="residential biomass",
-            p_set=p_set_biomass,
-        )
+    co2 = (
+        p_set_oil.sum().sum() * costs.at["solid biomass", "CO2 intensity"]
+    ) / 8760
 
-        n.madd(
-            "Load",
-            nodes,
-            suffix=" residential gas",
-            bus=spatial.gas.nodes,
-            carrier="residential gas",
-            p_set=p_set_gas,
-        )
-        co2 = (p_set_gas.sum().sum() * costs.at["gas", "CO2 intensity"]) / 8760
+    n.add(
+        "Load",
+        "residential biomass emissions",
+        bus="co2 atmosphere",
+        carrier="biomass emissions",
+        p_set=-co2,
+    )
 
-        n.add(
-            "Load",
-            "residential gas emissions",
-            bus="co2 atmosphere",
-            carrier="gas emissions",
-            p_set=-co2,
+    for country in countries:
+        rem_heat_demand = (
+            energy_totals.loc[country, "total residential space"]
+            + energy_totals.loc[country, "total residential water"]
+            - energy_totals.loc[country, "residential heat biomass"]
+            - energy_totals.loc[country, "residential heat oil"]
+            - energy_totals.loc[country, "residential heat gas"]
         )
 
-        co2 = (
-            p_set_oil.sum().sum() * costs.at["solid biomass", "CO2 intensity"]
-        ) / 8760
-
-        n.add(
-            "Load",
-            "residential biomass emissions",
-            bus="co2 atmosphere",
-            carrier="biomass emissions",
-            p_set=-co2,
+        heat_buses = (n.loads_t.p_set.filter(regex="heat")).columns
+        n.loads_t.p_set.loc[:, heat_buses] = (
+            (
+                n.loads_t.p_set.filter(like=country)[heat_buses]
+                / n.loads_t.p_set.filter(like=country)[heat_buses].sum().sum()
+            )
+            * rem_heat_demand
+            * 1e6
         )
 
-        for country in countries:
-            rem_heat_demand = (
-                energy_totals.loc[country, "total residential space"]
-                + energy_totals.loc[country, "total residential water"]
-                - energy_totals.loc[country, "residential heat biomass"]
-                - energy_totals.loc[country, "residential heat oil"]
-                - energy_totals.loc[country, "residential heat gas"]
-            )
+        # if snakemake.config["custom_data"]["elec_demand"]:
+    for country in countries:
+        # indd=n.loads_t.p_set[n.loads_t.p_set.columns.str.contains(country)]
 
-            heat_buses = (n.loads_t.p_set.filter(regex="heat")).columns
-            n.loads_t.p_set.loc[:, heat_buses] = (
-                (
-                    n.loads_t.p_set.filter(like=country)[heat_buses]
-                    / n.loads_t.p_set.filter(like=country)[heat_buses].sum().sum()
-                )
-                * rem_heat_demand
-                * 1e6
-            )
+        buses = n.buses[
+            (n.buses.carrier == "AC") & (n.buses.country == country)
+        ].index
 
-        if snakemake.config["custom_data"]["elec_demand"]:
-            for country in countries:
-                # indd=n.loads_t.p_set[n.loads_t.p_set.columns.str.contains(country)]
-
-                buses = n.buses[
-                    (n.buses.carrier == "AC") & (n.buses.country == country)
-                ].index
-
-                n.loads_t.p_set.loc[:, buses] = (
-                    (
-                        n.loads_t.p_set.filter(like=country)[buses]
-                        / n.loads_t.p_set.filter(like=country)[buses].sum().sum()
-                    )
-                    * energy_totals.loc[country, "electricity residential"]
-                    * 1e6
-                )
+        n.loads_t.p_set.loc[:, buses] = (
+            (
+                n.loads_t.p_set.filter(like=country)[buses]
+                / n.loads_t.p_set.filter(like=country)[buses].sum().sum()
+            )
+            * energy_totals.loc[country, "electricity residential"]
+            * 1e6
+        )
 
 
 # def add_co2limit(n, Nyears=1.0, limit=0.0):
diff --git a/scripts/prepare_transport_data.py b/scripts/prepare_transport_data.py
index a7a04a18..a1d61f93 100644
--- a/scripts/prepare_transport_data.py
+++ b/scripts/prepare_transport_data.py
@@ -206,7 +206,7 @@ def prepare_transport_data(n):
             "prepare_transport_data",
             simpl="",
             clusters="10",
-            demand="DF",
+            demand="AB",
             planning_horizons=2030,
         )
 

From 9acfc8d4005ac604a78a5b607f77153487f493f5 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 24 Aug 2023 14:30:38 +0200
Subject: [PATCH 32/58] add all sectors to the default workflow

---
 scripts/prepare_sector_network.py | 50 +++++++++++++++----------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 06b5f2c0..2c9ed9fa 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1296,24 +1296,24 @@ def add_industry(n, costs):
         if n.loads_t.p_set[loads_i].empty:
             continue
 
-        if not snakemake.config["custom_data"]["elec_demand"]:
-            # if electricity demand is provided by pypsa-earth, the electricty used
-            # in industry is included, and need to be removed from the default elec
-            # demand here, and added as "industry electricity"
-            factor = (
-                1
-                - industrial_demand.loc[loads_i, "current electricity"].sum()
-                / n.loads_t.p_set[loads_i].sum().sum()
-            )
-            n.loads_t.p_set[loads_i] *= factor
-            industrial_elec = industrial_demand["current electricity"].apply(
-                lambda frac: frac / 8760
-            )
+    # if not snakemake.config["custom_data"]["elec_demand"]:
+    #     # if electricity demand is provided by pypsa-earth, the electricty used
+    #     # in industry is included, and need to be removed from the default elec
+    #     # demand here, and added as "industry electricity"
+    #     factor = (
+    #         1
+    #         - industrial_demand.loc[loads_i, "current electricity"].sum()
+    #         / n.loads_t.p_set[loads_i].sum().sum()
+    #     )
+    #     n.loads_t.p_set[loads_i] *= factor
+    #     industrial_elec = industrial_demand["current electricity"].apply(
+    #         lambda frac: frac / 8760
+    #     )
 
-        else:
-            industrial_elec = industrial_demand["electricity"].apply(
-                lambda frac: frac / 8760
-            )
+    # else:
+    industrial_elec = industrial_demand["electricity"].apply(
+        lambda frac: frac / 8760
+    )
 
     n.madd(
         "Load",
@@ -2314,13 +2314,13 @@ def add_rail_transport(n, costs):
         snakemake = mock_snakemake(
             "prepare_sector_network",
             simpl="",
-            clusters="11",
+            clusters="14",
             ll="c1.0",
             opts="Co2L",
             planning_horizons="2030",
             sopts="24H",
             discountrate="0.071",
-            demand="DF",
+            demand="XX",
         )
 
     # Load population layout
@@ -2457,13 +2457,13 @@ def add_rail_transport(n, costs):
 
     add_land_transport(n, costs)
 
-    if snakemake.config["custom_data"]["transport_demand"]:
-        add_rail_transport(n, costs)
+    #if snakemake.config["custom_data"]["transport_demand"]:
+    add_rail_transport(n, costs)
 
-    if snakemake.config["custom_data"]["custom_sectors"]:
-        add_agriculture(n, costs)
-        add_residential(n, costs)
-        add_services(n, costs)
+    #if snakemake.config["custom_data"]["custom_sectors"]:
+    add_agriculture(n, costs)
+    add_residential(n, costs)
+    add_services(n, costs)
 
     sopts = snakemake.wildcards.sopts.split("-")
 

From cc14933c8a71e74b0f377337f954a4dfbb773ec8 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 24 Aug 2023 12:31:02 +0000
Subject: [PATCH 33/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_sector_network.py | 27 ++++++++-------------------
 1 file changed, 8 insertions(+), 19 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 2c9ed9fa..06e36ed0 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1311,9 +1311,7 @@ def add_industry(n, costs):
     #     )
 
     # else:
-    industrial_elec = industrial_demand["electricity"].apply(
-        lambda frac: frac / 8760
-    )
+    industrial_elec = industrial_demand["electricity"].apply(lambda frac: frac / 8760)
 
     n.madd(
         "Load",
@@ -2093,17 +2091,14 @@ def add_residential(n, costs):
     )
 
     heat_shape = (
-        n.loads_t.p_set.loc[:, heat_ind]
-        / n.loads_t.p_set.loc[:, heat_ind].sum().sum()
+        n.loads_t.p_set.loc[:, heat_ind] / n.loads_t.p_set.loc[:, heat_ind].sum().sum()
     )
     heat_oil_demand = (
         heat_shape * energy_totals.loc[countries[0], "residential heat oil"] * 1e6
     )
 
     heat_biomass_demand = (
-        heat_shape
-        * energy_totals.loc[countries[0], "residential heat biomass"]
-        * 1e6
+        heat_shape * energy_totals.loc[countries[0], "residential heat biomass"] * 1e6
     )
 
     heat_gas_demand = (
@@ -2123,9 +2118,7 @@ def add_residential(n, costs):
     )
 
     # TODO make compatible with more counties
-    profile_residential = (
-        n.loads_t.p_set[nodes] / n.loads_t.p_set[nodes].sum().sum()
-    )
+    profile_residential = n.loads_t.p_set[nodes] / n.loads_t.p_set[nodes].sum().sum()
 
     p_set_oil = (
         profile_residential
@@ -2189,9 +2182,7 @@ def add_residential(n, costs):
         p_set=-co2,
     )
 
-    co2 = (
-        p_set_oil.sum().sum() * costs.at["solid biomass", "CO2 intensity"]
-    ) / 8760
+    co2 = (p_set_oil.sum().sum() * costs.at["solid biomass", "CO2 intensity"]) / 8760
 
     n.add(
         "Load",
@@ -2224,9 +2215,7 @@ def add_residential(n, costs):
     for country in countries:
         # indd=n.loads_t.p_set[n.loads_t.p_set.columns.str.contains(country)]
 
-        buses = n.buses[
-            (n.buses.carrier == "AC") & (n.buses.country == country)
-        ].index
+        buses = n.buses[(n.buses.carrier == "AC") & (n.buses.country == country)].index
 
         n.loads_t.p_set.loc[:, buses] = (
             (
@@ -2457,10 +2446,10 @@ def add_rail_transport(n, costs):
 
     add_land_transport(n, costs)
 
-    #if snakemake.config["custom_data"]["transport_demand"]:
+    # if snakemake.config["custom_data"]["transport_demand"]:
     add_rail_transport(n, costs)
 
-    #if snakemake.config["custom_data"]["custom_sectors"]:
+    # if snakemake.config["custom_data"]["custom_sectors"]:
     add_agriculture(n, costs)
     add_residential(n, costs)
     add_services(n, costs)

From 4d15bdde6c39403bac7ff85579f9bfe7046b0b4c Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Fri, 25 Aug 2023 11:39:10 +0200
Subject: [PATCH 34/58] fix bug of heat shape

---
 scripts/prepare_sector_network.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 06e36ed0..93289cb9 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -2093,6 +2093,9 @@ def add_residential(n, costs):
     heat_shape = (
         n.loads_t.p_set.loc[:, heat_ind] / n.loads_t.p_set.loc[:, heat_ind].sum().sum()
     )
+
+    heat_shape = heat_shape.groupby(lambda x: next((substring for substring in nodes if substring in x), x), axis=1).sum()
+    
     heat_oil_demand = (
         heat_shape * energy_totals.loc[countries[0], "residential heat oil"] * 1e6
     )

From b5ed6dd5ed141ea38dd9fc78451074567d4b5951 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 25 Aug 2023 09:40:00 +0000
Subject: [PATCH 35/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_sector_network.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 93289cb9..c34aad47 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -2094,8 +2094,10 @@ def add_residential(n, costs):
         n.loads_t.p_set.loc[:, heat_ind] / n.loads_t.p_set.loc[:, heat_ind].sum().sum()
     )
 
-    heat_shape = heat_shape.groupby(lambda x: next((substring for substring in nodes if substring in x), x), axis=1).sum()
-    
+    heat_shape = heat_shape.groupby(
+        lambda x: next((substring for substring in nodes if substring in x), x), axis=1
+    ).sum()
+
     heat_oil_demand = (
         heat_shape * energy_totals.loc[countries[0], "residential heat oil"] * 1e6
     )

From ed04065afa0736e73c1872d8f4639d300bedf6e3 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Fri, 25 Aug 2023 12:46:45 +0200
Subject: [PATCH 36/58] diagnosing CI bug

---
 scripts/prepare_sector_network.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index c34aad47..93efc344 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -2121,6 +2121,18 @@ def add_residential(n, costs):
         )
         * 1e6
     )
+    
+    print('#############################################')
+    print('#############################################')
+    print('#############################################')
+    print(nodes)
+    print('#############################################')
+    print('#############################################')
+    print('#############################################')
+    print(heat_shape)
+    print('#############################################')
+    print('#############################################')
+    print('#############################################')
 
     # TODO make compatible with more counties
     profile_residential = n.loads_t.p_set[nodes] / n.loads_t.p_set[nodes].sum().sum()

From 32ce03db1620659696c1e08e621d6d35af5e2cd1 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 25 Aug 2023 10:50:19 +0000
Subject: [PATCH 37/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_sector_network.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 93efc344..b552ccb9 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -2121,18 +2121,18 @@ def add_residential(n, costs):
         )
         * 1e6
     )
-    
-    print('#############################################')
-    print('#############################################')
-    print('#############################################')
+
+    print("#############################################")
+    print("#############################################")
+    print("#############################################")
     print(nodes)
-    print('#############################################')
-    print('#############################################')
-    print('#############################################')
+    print("#############################################")
+    print("#############################################")
+    print("#############################################")
     print(heat_shape)
-    print('#############################################')
-    print('#############################################')
-    print('#############################################')
+    print("#############################################")
+    print("#############################################")
+    print("#############################################")
 
     # TODO make compatible with more counties
     profile_residential = n.loads_t.p_set[nodes] / n.loads_t.p_set[nodes].sum().sum()

From e1e8a697d0f82d6f74f355b45af00c9c7e358ac9 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Fri, 25 Aug 2023 13:28:31 +0200
Subject: [PATCH 38/58] CI bug solving trial

---
 scripts/prepare_sector_network.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index b552ccb9..f8bc9086 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -2061,30 +2061,26 @@ def add_residential(n, costs):
     # oil_res_index=n.loads_t.p.filter(like='residential').filter(like='oil').dropna(axis=1).index
 
     heat_ind = (
-        n.loads_t.p_set.filter(like=countries[0])
-        .filter(like="residential")
+        n.loads_t.p_set.filter(like="residential")
         .filter(like="heat")
         .dropna(axis=1)
         .columns
     )
     oil_ind = (
-        n.loads_t.p_set.filter(like=countries[0])
-        .filter(like="residential")
+        n.loads_t.p_set.filter(like="residential")
         .filter(like="oil")
         .dropna(axis=1)
         .columns
     )
     bio_ind = (
-        n.loads_t.p_set.filter(like=countries[0])
-        .filter(like="residential")
+        n.loads_t.p_set.filter(like="residential")
         .filter(like="biomass")
         .dropna(axis=1)
         .columns
     )
 
     gas_ind = (
-        n.loads_t.p_set.filter(like=countries[0])
-        .filter(like="residential")
+        n.loads_t.p_set.filter(like="residential")
         .filter(like="gas")
         .dropna(axis=1)
         .columns

From 3258cd702aa0e21f190ddf60fe6fca20b7932a8b Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Fri, 25 Aug 2023 14:38:01 +0200
Subject: [PATCH 39/58] =?UTF-8?q?make=20p=C3=B6d=20custom=20sector=20work?=
 =?UTF-8?q?=20for=20many=20countries=20at=20once?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/prepare_sector_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index f8bc9086..87684f29 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -2214,7 +2214,7 @@ def add_residential(n, costs):
             - energy_totals.loc[country, "residential heat gas"]
         )
 
-        heat_buses = (n.loads_t.p_set.filter(regex="heat")).columns
+        heat_buses = (n.loads_t.p_set.filter(regex="heat").filter(like=country)).columns
         n.loads_t.p_set.loc[:, heat_buses] = (
             (
                 n.loads_t.p_set.filter(like=country)[heat_buses]

From 174ae272e7e63c5783b3d8937f55a314fce2c7ce Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Tue, 29 Aug 2023 10:25:04 +0200
Subject: [PATCH 40/58] resolving PR comments

---
 .gitignore                            | 2 ++
 config.default.yaml                   | 5 +++--
 data/demand/efficiency_gains copy.csv | 2 --
 data/demand/growth_factors copy.csv   | 2 --
 data/energy_totals_XX_2030.csv        | 2 --
 data/energy_totals_XX_2030_.csv       | 2 --
 scripts/build_base_energy_totals.py   | 8 ++++----
 test/config.test1.yaml                | 3 ++-
 8 files changed, 11 insertions(+), 15 deletions(-)
 delete mode 100644 data/demand/efficiency_gains copy.csv
 delete mode 100644 data/demand/growth_factors copy.csv
 delete mode 100644 data/energy_totals_XX_2030.csv
 delete mode 100644 data/energy_totals_XX_2030_.csv

diff --git a/.gitignore b/.gitignore
index 4229db5b..0ea1d517 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,6 +39,8 @@ data/industry_sector_ratios_NZ_2030.csv
 /data/nuts*
 /data/demand/unsd/data
 /data/ports.csv
+/data/energy_totals_DF_2030
+/data/energy_totals_XX_2038
 
 /data/airports.csv
 /data/energy_totals_base.csv
diff --git a/config.default.yaml b/config.default.yaml
index 131b489c..26198dad 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -40,7 +40,7 @@ clustering_options:
 countries: ['MA']
 
 demand_data:
-  update_data: true
+  update_data: true # if true, the workflow downloads the energy balances data saved in data/demand/unsd/data again. Turn on for the first run.
   base_year: 2019
 
 H2_network: false
@@ -270,7 +270,8 @@ sector:
   SMR: true
   cc_fraction: 0.9
   cc: true
-
+  space_heat_share: 0.6 # the share of space heating from all heating. Remainder goes to water heating.
+  
   conventional_generation: # generator : carrier
     OCGT: gas
     #Gen_Test: oil # Just for testing purposes
diff --git a/data/demand/efficiency_gains copy.csv b/data/demand/efficiency_gains copy.csv
deleted file mode 100644
index 43fc4975..00000000
--- a/data/demand/efficiency_gains copy.csv	
+++ /dev/null
@@ -1,2 +0,0 @@
-,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen
-MA,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,0.95,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85
diff --git a/data/demand/growth_factors copy.csv b/data/demand/growth_factors copy.csv
deleted file mode 100644
index 8775d5f6..00000000
--- a/data/demand/growth_factors copy.csv	
+++ /dev/null
@@ -1,2 +0,0 @@
-,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas
-MA,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.132,1.41,1.23,1.32,0.5,1.0,0.89
diff --git a/data/energy_totals_XX_2030.csv b/data/energy_totals_XX_2030.csv
deleted file mode 100644
index 807e8a0b..00000000
--- a/data/energy_totals_XX_2030.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
-MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,33.67658116,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.693556480000002,1.9635325499999998,0.1264536,0.0,0.0,85.63625160000001,0.0,0.0,0,0,0,0,0
diff --git a/data/energy_totals_XX_2030_.csv b/data/energy_totals_XX_2030_.csv
deleted file mode 100644
index 4e454a61..00000000
--- a/data/energy_totals_XX_2030_.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
-MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.693556480000002,1.40252325,0.1264536,0.0,0.0,71.27159790000002,0.0,0.0,0,0,0,0,0
diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index 35794ee8..1c2e8633 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -97,13 +97,13 @@ def calc_sector(sector):
                     df_sector[df_sector.Commodity.isin(heat)]
                     .Quantity_TWh.sum()
                     .round(4)
-                    * 0.6
+                    * snakemake.config["sector"]["space_heat_share"]
                 )
                 energy_totals_base.at[country, "total residential water"] = (
                     df_sector[df_sector.Commodity.isin(heat)]
                     .Quantity_TWh.sum()
                     .round(4)
-                    * 0.4
+                    * (1 - snakemake.config["sector"]["space_heat_share"])
                 )
 
             elif sector == "services":
@@ -134,13 +134,13 @@ def calc_sector(sector):
                     df_sector[df_sector.Commodity.isin(heat)]
                     .Quantity_TWh.sum()
                     .round(4)
-                    * 0.6
+                    * snakemake.config["sector"]["space_heat_share"]
                 )
                 energy_totals_base.at[country, "total services water"] = (
                     df_sector[df_sector.Commodity.isin(heat)]
                     .Quantity_TWh.sum()
                     .round(4)
-                    * 0.4
+                    * (1 - snakemake.config["sector"]["space_heat_share"])
                 )
 
             elif sector == "road":
diff --git a/test/config.test1.yaml b/test/config.test1.yaml
index a8f4d1c7..9d418201 100644
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@@ -41,7 +41,7 @@ clustering_options:
 countries: ['NG', 'BJ']
 
 demand_data:
-  update_data: true
+  update_data: true # if true, the workflow downloads the energy balances data saved in data/demand/unsd/data again. Turn on for the first run.
   base_year: 2019
 
 H2_network: false
@@ -274,6 +274,7 @@ sector:
   SMR: true
   cc_fraction: 0.9
   cc: true
+  space_heat_share: 0.6 # the share of space heating from all heating. Remainder goes to water heating.
 
   conventional_generation: # generator : carrier
     OCGT: gas

From 1614c58752d025f57006d1d9bd026755550c1955 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 29 Aug 2023 08:25:25 +0000
Subject: [PATCH 41/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 config.default.yaml                 |  1 -
 scripts/build_base_energy_totals.py | 18 ++++++++----------
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index 26198dad..b3791230 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -271,7 +271,6 @@ sector:
   cc_fraction: 0.9
   cc: true
   space_heat_share: 0.6 # the share of space heating from all heating. Remainder goes to water heating.
-  
   conventional_generation: # generator : carrier
     OCGT: gas
     #Gen_Test: oil # Just for testing purposes
diff --git a/scripts/build_base_energy_totals.py b/scripts/build_base_energy_totals.py
index 1c2e8633..273b3d58 100644
--- a/scripts/build_base_energy_totals.py
+++ b/scripts/build_base_energy_totals.py
@@ -99,11 +99,10 @@ def calc_sector(sector):
                     .round(4)
                     * snakemake.config["sector"]["space_heat_share"]
                 )
-                energy_totals_base.at[country, "total residential water"] = (
-                    df_sector[df_sector.Commodity.isin(heat)]
-                    .Quantity_TWh.sum()
-                    .round(4)
-                    * (1 - snakemake.config["sector"]["space_heat_share"])
+                energy_totals_base.at[country, "total residential water"] = df_sector[
+                    df_sector.Commodity.isin(heat)
+                ].Quantity_TWh.sum().round(4) * (
+                    1 - snakemake.config["sector"]["space_heat_share"]
                 )
 
             elif sector == "services":
@@ -136,11 +135,10 @@ def calc_sector(sector):
                     .round(4)
                     * snakemake.config["sector"]["space_heat_share"]
                 )
-                energy_totals_base.at[country, "total services water"] = (
-                    df_sector[df_sector.Commodity.isin(heat)]
-                    .Quantity_TWh.sum()
-                    .round(4)
-                    * (1 - snakemake.config["sector"]["space_heat_share"])
+                energy_totals_base.at[country, "total services water"] = df_sector[
+                    df_sector.Commodity.isin(heat)
+                ].Quantity_TWh.sum().round(4) * (
+                    1 - snakemake.config["sector"]["space_heat_share"]
                 )
 
             elif sector == "road":

From 7732e1b81c5d77419470ed56c33095478ab0a7bd Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Tue, 29 Aug 2023 10:30:59 +0200
Subject: [PATCH 42/58] resemove unnecessary files

---
 data/energy_totals_DF_2030.csv | 2 --
 data/energy_totals_XX_2038.csv | 2 --
 2 files changed, 4 deletions(-)
 delete mode 100644 data/energy_totals_DF_2030.csv
 delete mode 100644 data/energy_totals_XX_2038.csv

diff --git a/data/energy_totals_DF_2030.csv b/data/energy_totals_DF_2030.csv
deleted file mode 100644
index 4ffa5e6b..00000000
--- a/data/energy_totals_DF_2030.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
-MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.693556480000002,1.9635325499999998,0.1264536,0.0,0.0,85.52591748000002,0.0,0.0,0,0,0,0,0
diff --git a/data/energy_totals_XX_2038.csv b/data/energy_totals_XX_2038.csv
deleted file mode 100644
index b9645598..00000000
--- a/data/energy_totals_XX_2038.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential heat biomass,residential heat gas,residential heat oil,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,electricity residential space,electricity residential water,district heat share,electricity services space,electricity services water
-MA,0.0,4.7861475,6.9489728,0.7777727999999999,35.97336541,1.250795,0.0,3.917325,0.0,9.917039160499998,5.218711410000001,6.91083,11.17675,0.0,0.31075,0.792,0.0,14.2347,2.40155736,1.1411400000000003,13.937672334899998,9.2917815566,73.2802447,0.0,0.0,5.6370538386,3.7580358924,0,0,0

From bf8833ab4aa17cdbe67266bb34c6c74013d49503 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Tue, 29 Aug 2023 11:37:50 +0200
Subject: [PATCH 43/58] add example energy_totals file needed for workflow

---
 data/energy_totals_DF_2030.csv | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 data/energy_totals_DF_2030.csv

diff --git a/data/energy_totals_DF_2030.csv b/data/energy_totals_DF_2030.csv
new file mode 100644
index 00000000..a0af1365
--- /dev/null
+++ b/data/energy_totals_DF_2030.csv
@@ -0,0 +1,2 @@
+,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
+MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.69355680000002,1.9635325499999998,0.1264536,0.0,0.0,85.52591748000002,0.0,0.0,0,0,0,0,0
\ No newline at end of file

From 243f9d1e889beb98f33e0af6103c335c438e3568 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 29 Aug 2023 09:39:52 +0000
Subject: [PATCH 44/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 data/energy_totals_DF_2030.csv | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data/energy_totals_DF_2030.csv b/data/energy_totals_DF_2030.csv
index a0af1365..0cccb008 100644
--- a/data/energy_totals_DF_2030.csv
+++ b/data/energy_totals_DF_2030.csv
@@ -1,2 +1,2 @@
 ,agriculture biomass,agriculture electricity,agriculture oil,electricity rail,electricity residential,residential biomass,residential gas,residential oil,services biomass,services electricity,services gas,services oil,total domestic aviation,total domestic navigation,total international aviation,total international navigation,total rail,total residential space,total residential water,total road,total services space,total services water,district heat share,electricity residential water,electricity residential space,electricity services space,electricity services water
-MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.69355680000002,1.9635325499999998,0.1264536,0.0,0.0,85.52591748000002,0.0,0.0,0,0,0,0,0
\ No newline at end of file
+MA,0.0,3.7943233899999993,12.220265024000001,0.4112628,11.8740713,7.0544838,0.0,25.168813287999996,4.96120807,5.9330894999999995,0.0,0.765688,0.3847932,0.0,12.69355680000002,1.9635325499999998,0.1264536,0.0,0.0,85.52591748000002,0.0,0.0,0,0,0,0,0

From d9cfc2db9a3b77f996fd2a4f844868dbd73f18bc Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Tue, 29 Aug 2023 17:58:08 +0200
Subject: [PATCH 45/58] change growth factors and efficiency gains to cagr

---
 data/demand/efficiency_gains.csv      |  4 ----
 data/demand/efficiency_gains_cagr.csv |  5 +++++
 data/demand/growth_factors.csv        |  4 ----
 data/demand/growth_factors_cagr.csv   |  5 +++++
 scripts/prepare_energy_totals.py      | 15 ++++++++++-----
 5 files changed, 20 insertions(+), 13 deletions(-)
 delete mode 100644 data/demand/efficiency_gains.csv
 create mode 100644 data/demand/efficiency_gains_cagr.csv
 delete mode 100644 data/demand/growth_factors.csv
 create mode 100644 data/demand/growth_factors_cagr.csv

diff --git a/data/demand/efficiency_gains.csv b/data/demand/efficiency_gains.csv
deleted file mode 100644
index e55aa271..00000000
--- a/data/demand/efficiency_gains.csv
+++ /dev/null
@@ -1,4 +0,0 @@
-,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen,residential heat biomass,residential heat oil,residential heat gas
-MA,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,1.0,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85, 0.95, 0.95, 1.0
-NG,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,1.0,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85, 0.95, 0.95, 1.0
-BJ,0.91,0.82,0.95,0.88,0.71,0.91,0.96,1.0,1.0,0.71,100,0.95,0.95,0.88,1.0,0.91,0.82,0.79,1.0,0.9,1.0, 0.3, 0.6, 1.0, 0.98, 0.85, 0.95, 0.95, 1.0
diff --git a/data/demand/efficiency_gains_cagr.csv b/data/demand/efficiency_gains_cagr.csv
new file mode 100644
index 00000000..44036593
--- /dev/null
+++ b/data/demand/efficiency_gains_cagr.csv
@@ -0,0 +1,5 @@
+,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen,residential heat biomass,residential heat oil,residential heat gas
+MA,-0.00853704879880779,-0.017879229947568076,-0.004652171732197341,-0.011553950087018094,-0.03065576512859003,-0.00853704879880779,-0.0037042128253242934,0.0,0.0,-0.03065576512859003,0.5199110829529336,-0.004652171732197341,-0.004652171732197341,-0.011553950087018094,0.0,-0.00853704879880779,-0.017879229947568076,-0.021201326893414696,0.0,-0.009532503568715489,0.0,-0.10367487860124702,-0.04537691621306539,0.0,-0.0018349242204882543,-0.014665841502869426,-0.004652171732197341,-0.004652171732197341,0.0
+NG,-0.00853704879880779,-0.017879229947568076,-0.004652171732197341,-0.011553950087018094,-0.03065576512859003,-0.00853704879880779,-0.0037042128253242934,0.0,0.0,-0.03065576512859003,0.5199110829529336,-0.004652171732197341,-0.004652171732197341,-0.011553950087018094,0.0,-0.00853704879880779,-0.017879229947568076,-0.021201326893414696,0.0,-0.009532503568715489,0.0,-0.10367487860124702,-0.04537691621306539,0.0,-0.0018349242204882543,-0.014665841502869426,-0.004652171732197341,-0.004652171732197341,0.0
+BJ,-0.00853704879880779,-0.017879229947568076,-0.004652171732197341,-0.011553950087018094,-0.03065576512859003,-0.00853704879880779,-0.0037042128253242934,0.0,0.0,-0.03065576512859003,0.5199110829529336,-0.004652171732197341,-0.004652171732197341,-0.011553950087018094,0.0,-0.00853704879880779,-0.017879229947568076,-0.021201326893414696,0.0,-0.009532503568715489,0.0,-0.10367487860124702,-0.04537691621306539,0.0,-0.0018349242204882543,-0.014665841502869426,-0.004652171732197341,-0.004652171732197341,0.0
+EG,-0.00853704879880779,-0.017879229947568076,-0.004652171732197341,-0.011553950087018094,-0.03065576512859003,-0.00853704879880779,-0.0037042128253242934,0.0,0.0,-0.03065576512859003,0.5199110829529336,-0.004652171732197341,-0.004652171732197341,-0.011553950087018094,0.0,-0.00853704879880779,-0.017879229947568076,-0.021201326893414696,0.0,-0.009532503568715489,0.0,-0.10367487860124702,-0.04537691621306539,0.0,-0.0018349242204882543,-0.014665841502869426,-0.004652171732197341,-0.004652171732197341,0.0
diff --git a/data/demand/growth_factors.csv b/data/demand/growth_factors.csv
deleted file mode 100644
index 6e09e3b9..00000000
--- a/data/demand/growth_factors.csv
+++ /dev/null
@@ -1,4 +0,0 @@
-,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,residential heat oil,residential heat biomass,residential heat gas
-MA,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.0,1.0,1.23,1.32,0.5,1.0,0.89, 1.0, 1.0, 1.0
-NG,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.0,1.0,1.23,1.32,0.5,1.0,0.89, 1.0, 1.0, 1.0
-BJ,1.5,1.6,2.8,1.4,1.5,1.3,2.5,2.1,2.0,1.5,1.13,1.6,2.0,1.5,0.7,1.0,1.0,1.23,1.32,0.5,1.0,0.89, 1.0, 1.0, 1.0
diff --git a/data/demand/growth_factors_cagr.csv b/data/demand/growth_factors_cagr.csv
new file mode 100644
index 00000000..855099c9
--- /dev/null
+++ b/data/demand/growth_factors_cagr.csv
@@ -0,0 +1,5 @@
+,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,residential heat oil,residential heat biomass,residential heat gas
+MA,0.03754823579391897,0.04365356767031847,0.09812234829384714,0.031061016501513494,0.03754823579391897,0.02413801393598547,0.08686690424248389,0.06977553866703001,0.06504108943996267,0.03754823579391897,0.011172646877630532,0.04365356767031847,0.06504108943996267,0.03754823579391897,-0.03190494082844941,0.0,0.0,0.018997672304465807,0.025560455264682425,-0.06106908933829369,0.0,-0.010538064696069105,0.0,0.0,0.0
+NG,0.03754823579391897,0.04365356767031847,0.09812234829384714,0.031061016501513494,0.03754823579391897,0.02413801393598547,0.08686690424248389,0.06977553866703001,0.06504108943996267,0.03754823579391897,0.011172646877630532,0.04365356767031847,0.06504108943996267,0.03754823579391897,-0.03190494082844941,0.0,0.0,0.018997672304465807,0.025560455264682425,-0.06106908933829369,0.0,-0.010538064696069105,0.0,0.0,0.0
+BJ,0.03754823579391897,0.04365356767031847,0.09812234829384714,0.031061016501513494,0.03754823579391897,0.02413801393598547,0.08686690424248389,0.06977553866703001,0.06504108943996267,0.03754823579391897,0.011172646877630532,0.04365356767031847,0.06504108943996267,0.03754823579391897,-0.03190494082844941,0.0,0.0,0.018997672304465807,0.025560455264682425,-0.06106908933829369,0.0,-0.010538064696069105,0.0,0.0,0.0
+EG,0.03754823579391897,0.04365356767031847,0.09812234829384714,0.031061016501513494,0.03754823579391897,0.02413801393598547,0.08686690424248389,0.06977553866703001,0.06504108943996267,0.03754823579391897,0.011172646877630532,0.04365356767031847,0.06504108943996267,0.03754823579391897,-0.03190494082844941,0.0,0.0,0.018997672304465807,0.025560455264682425,-0.06106908933829369,0.0,-0.010538064696069105,0.0,0.0,0.0
diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index e3faa738..0570324f 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -23,6 +23,9 @@ def get(item, investment_year=None):
     else:
         return item
 
+def calculate_end_values(df):
+    return (1 + df) ** no_years
+
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
@@ -33,8 +36,8 @@ def get(item, investment_year=None):
         snakemake = mock_snakemake(
             "prepare_energy_totals",
             simpl="",
-            clusters=10,
-            demand="AB",
+            clusters=32,
+            demand="EG",
             planning_horizons=2030,
         )
         sets_path_to_root("pypsa-earth-sec")
@@ -45,12 +48,14 @@ def get(item, investment_year=None):
     demand_sc = snakemake.wildcards.demand  # loading the demand scenrario wildcard
 
     base_energy_totals = pd.read_csv("data/energy_totals_base.csv", index_col=0)
-    growth_factors = pd.read_csv("data/demand/growth_factors.csv", index_col=0)
-    efficiency_gains = pd.read_csv("data/demand/efficiency_gains.csv", index_col=0)
+    growth_factors_cagr = pd.read_csv("data/demand/growth_factors_cagr.csv", index_col=0)
+    efficiency_gains_cagr = pd.read_csv("data/demand/efficiency_gains_cagr.csv", index_col=0)
     fuel_shares = pd.read_csv("data/demand/fuel_shares.csv", index_col=0)
     district_heating = pd.read_csv("data/demand/district_heating.csv", index_col=0)
 
-    # growth_factors = growth_factors[growth_factors.index.isin(countries)]
+    no_years = int(snakemake.wildcards.planning_horizons) - int(snakemake.config["demand_data"]["base_year"])
+    growth_factors = calculate_end_values(growth_factors_cagr)
+    efficiency_gains = calculate_end_values(efficiency_gains_cagr)
     # efficiency_gains = efficiency_gains[efficiency_gains.index.isin(countries)]
     # fuel_shares = fuel_shares[fuel_shares.index.isin(countries)]
     # district_heating = district_heating[district_heating.index.isin(countries)]

From 2b5a0a9b5ded38f2bf013dd7d23c4d621d8636bc Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 29 Aug 2023 15:58:24 +0000
Subject: [PATCH 46/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_energy_totals.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index 0570324f..c4ba94bc 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -23,6 +23,7 @@ def get(item, investment_year=None):
     else:
         return item
 
+
 def calculate_end_values(df):
     return (1 + df) ** no_years
 
@@ -48,12 +49,18 @@ def calculate_end_values(df):
     demand_sc = snakemake.wildcards.demand  # loading the demand scenrario wildcard
 
     base_energy_totals = pd.read_csv("data/energy_totals_base.csv", index_col=0)
-    growth_factors_cagr = pd.read_csv("data/demand/growth_factors_cagr.csv", index_col=0)
-    efficiency_gains_cagr = pd.read_csv("data/demand/efficiency_gains_cagr.csv", index_col=0)
+    growth_factors_cagr = pd.read_csv(
+        "data/demand/growth_factors_cagr.csv", index_col=0
+    )
+    efficiency_gains_cagr = pd.read_csv(
+        "data/demand/efficiency_gains_cagr.csv", index_col=0
+    )
     fuel_shares = pd.read_csv("data/demand/fuel_shares.csv", index_col=0)
     district_heating = pd.read_csv("data/demand/district_heating.csv", index_col=0)
 
-    no_years = int(snakemake.wildcards.planning_horizons) - int(snakemake.config["demand_data"]["base_year"])
+    no_years = int(snakemake.wildcards.planning_horizons) - int(
+        snakemake.config["demand_data"]["base_year"]
+    )
     growth_factors = calculate_end_values(growth_factors_cagr)
     efficiency_gains = calculate_end_values(efficiency_gains_cagr)
     # efficiency_gains = efficiency_gains[efficiency_gains.index.isin(countries)]

From 243b8b5cd566b42affe7332d0af1e383814364b9 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Tue, 29 Aug 2023 22:25:39 +0200
Subject: [PATCH 47/58] correct the default cagr values

---
 data/demand/efficiency_gains_cagr.csv | 8 ++++----
 data/demand/growth_factors_cagr.csv   | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/data/demand/efficiency_gains_cagr.csv b/data/demand/efficiency_gains_cagr.csv
index 44036593..8c9d8680 100644
--- a/data/demand/efficiency_gains_cagr.csv
+++ b/data/demand/efficiency_gains_cagr.csv
@@ -1,5 +1,5 @@
 ,total residential space,total residential water,electricity residential,total services space,total services water,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,total road ev,total road fcev,total road ice,total navigation oil,total navigation hydrogen,residential heat biomass,residential heat oil,residential heat gas
-MA,-0.00853704879880779,-0.017879229947568076,-0.004652171732197341,-0.011553950087018094,-0.03065576512859003,-0.00853704879880779,-0.0037042128253242934,0.0,0.0,-0.03065576512859003,0.5199110829529336,-0.004652171732197341,-0.004652171732197341,-0.011553950087018094,0.0,-0.00853704879880779,-0.017879229947568076,-0.021201326893414696,0.0,-0.009532503568715489,0.0,-0.10367487860124702,-0.04537691621306539,0.0,-0.0018349242204882543,-0.014665841502869426,-0.004652171732197341,-0.004652171732197341,0.0
-NG,-0.00853704879880779,-0.017879229947568076,-0.004652171732197341,-0.011553950087018094,-0.03065576512859003,-0.00853704879880779,-0.0037042128253242934,0.0,0.0,-0.03065576512859003,0.5199110829529336,-0.004652171732197341,-0.004652171732197341,-0.011553950087018094,0.0,-0.00853704879880779,-0.017879229947568076,-0.021201326893414696,0.0,-0.009532503568715489,0.0,-0.10367487860124702,-0.04537691621306539,0.0,-0.0018349242204882543,-0.014665841502869426,-0.004652171732197341,-0.004652171732197341,0.0
-BJ,-0.00853704879880779,-0.017879229947568076,-0.004652171732197341,-0.011553950087018094,-0.03065576512859003,-0.00853704879880779,-0.0037042128253242934,0.0,0.0,-0.03065576512859003,0.5199110829529336,-0.004652171732197341,-0.004652171732197341,-0.011553950087018094,0.0,-0.00853704879880779,-0.017879229947568076,-0.021201326893414696,0.0,-0.009532503568715489,0.0,-0.10367487860124702,-0.04537691621306539,0.0,-0.0018349242204882543,-0.014665841502869426,-0.004652171732197341,-0.004652171732197341,0.0
-EG,-0.00853704879880779,-0.017879229947568076,-0.004652171732197341,-0.011553950087018094,-0.03065576512859003,-0.00853704879880779,-0.0037042128253242934,0.0,0.0,-0.03065576512859003,0.5199110829529336,-0.004652171732197341,-0.004652171732197341,-0.011553950087018094,0.0,-0.00853704879880779,-0.017879229947568076,-0.021201326893414696,0.0,-0.009532503568715489,0.0,-0.10367487860124702,-0.04537691621306539,0.0,-0.0018349242204882543,-0.014665841502869426,-0.004652171732197341,-0.004652171732197341,0.0
+MA,-0.004951421072036277,-0.010390428883694458,-0.0026960063028712566,-0.006705489381695462,-0.01786431270106026,-0.004951421072036277,-0.0021462195974406573,0.0,0.0,-0.01786431270106026,0.2742749857031337,-0.0026960063028712566,-0.0026960063028712566,-0.006705489381695462,0.0,-0.004951421072036277,-0.010390428883694458,-0.012329796028156648,0.0,-0.0055299435560203225,0.0,-0.06140104567447868,-0.026527359809364603,0.0,-0.0010627352815874014,-0.00851714967450834,-0.0026960063028712566,-0.0026960063028712566,0.0
+NG,-0.004951421072036277,-0.010390428883694458,-0.0026960063028712566,-0.006705489381695462,-0.01786431270106026,-0.004951421072036277,-0.0021462195974406573,0.0,0.0,-0.01786431270106026,0.2742749857031337,-0.0026960063028712566,-0.0026960063028712566,-0.006705489381695462,0.0,-0.004951421072036277,-0.010390428883694458,-0.012329796028156648,0.0,-0.0055299435560203225,0.0,-0.06140104567447868,-0.026527359809364603,0.0,-0.0010627352815874014,-0.00851714967450834,-0.0026960063028712566,-0.0026960063028712566,0.0
+BJ,-0.004951421072036277,-0.010390428883694458,-0.0026960063028712566,-0.006705489381695462,-0.01786431270106026,-0.004951421072036277,-0.0021462195974406573,0.0,0.0,-0.01786431270106026,0.2742749857031337,-0.0026960063028712566,-0.0026960063028712566,-0.006705489381695462,0.0,-0.004951421072036277,-0.010390428883694458,-0.012329796028156648,0.0,-0.0055299435560203225,0.0,-0.06140104567447868,-0.026527359809364603,0.0,-0.0010627352815874014,-0.00851714967450834,-0.0026960063028712566,-0.0026960063028712566,0.0
+EG,-0.004951421072036277,-0.010390428883694458,-0.0026960063028712566,-0.006705489381695462,-0.01786431270106026,-0.004951421072036277,-0.0021462195974406573,0.0,0.0,-0.01786431270106026,0.2742749857031337,-0.0026960063028712566,-0.0026960063028712566,-0.006705489381695462,0.0,-0.004951421072036277,-0.010390428883694458,-0.012329796028156648,0.0,-0.0055299435560203225,0.0,-0.06140104567447868,-0.026527359809364603,0.0,-0.0010627352815874014,-0.00851714967450834,-0.0026960063028712566,-0.0026960063028712566,0.0
diff --git a/data/demand/growth_factors_cagr.csv b/data/demand/growth_factors_cagr.csv
index 855099c9..f857d5fe 100644
--- a/data/demand/growth_factors_cagr.csv
+++ b/data/demand/growth_factors_cagr.csv
@@ -1,5 +1,5 @@
 ,total residential space,total residential water,electricity residential,total services space,total services water,total road,total rail,electricity rail,total domestic aviation,total international aviation,total domestic navigation,total international navigation,services electricity,agriculture electricity,agriculture oil,residential oil,residential biomass,residential gas,agriculture biomass,services oil,services biomass,services gas,residential heat oil,residential heat biomass,residential heat gas
-MA,0.03754823579391897,0.04365356767031847,0.09812234829384714,0.031061016501513494,0.03754823579391897,0.02413801393598547,0.08686690424248389,0.06977553866703001,0.06504108943996267,0.03754823579391897,0.011172646877630532,0.04365356767031847,0.06504108943996267,0.03754823579391897,-0.03190494082844941,0.0,0.0,0.018997672304465807,0.025560455264682425,-0.06106908933829369,0.0,-0.010538064696069105,0.0,0.0,0.0
-NG,0.03754823579391897,0.04365356767031847,0.09812234829384714,0.031061016501513494,0.03754823579391897,0.02413801393598547,0.08686690424248389,0.06977553866703001,0.06504108943996267,0.03754823579391897,0.011172646877630532,0.04365356767031847,0.06504108943996267,0.03754823579391897,-0.03190494082844941,0.0,0.0,0.018997672304465807,0.025560455264682425,-0.06106908933829369,0.0,-0.010538064696069105,0.0,0.0,0.0
-BJ,0.03754823579391897,0.04365356767031847,0.09812234829384714,0.031061016501513494,0.03754823579391897,0.02413801393598547,0.08686690424248389,0.06977553866703001,0.06504108943996267,0.03754823579391897,0.011172646877630532,0.04365356767031847,0.06504108943996267,0.03754823579391897,-0.03190494082844941,0.0,0.0,0.018997672304465807,0.025560455264682425,-0.06106908933829369,0.0,-0.010538064696069105,0.0,0.0,0.0
-EG,0.03754823579391897,0.04365356767031847,0.09812234829384714,0.031061016501513494,0.03754823579391897,0.02413801393598547,0.08686690424248389,0.06977553866703001,0.06504108943996267,0.03754823579391897,0.011172646877630532,0.04365356767031847,0.06504108943996267,0.03754823579391897,-0.03190494082844941,0.0,0.0,0.018997672304465807,0.025560455264682425,-0.06106908933829369,0.0,-0.010538064696069105,0.0,0.0,0.0
+MA,0.021569600815111167,0.025045532052292208,0.055685686523292866,0.01786680031720378,0.021569600815111167,0.01390442519840307,0.04940761418323669,0.03982178088300037,0.03715504444619189,0.021569600815111167,0.006453239989770987,0.025045532052292208,0.03715504444619189,0.021569600815111167,-0.018597262031625594,0.0,0.0,0.01095505452809764,0.0147194767041825,-0.03582400205750491,0.0,-0.006114588100848528,0.0,0.0,0.0
+NG,0.021569600815111167,0.025045532052292208,0.055685686523292866,0.01786680031720378,0.021569600815111167,0.01390442519840307,0.04940761418323669,0.03982178088300037,0.03715504444619189,0.021569600815111167,0.006453239989770987,0.025045532052292208,0.03715504444619189,0.021569600815111167,-0.018597262031625594,0.0,0.0,0.01095505452809764,0.0147194767041825,-0.03582400205750491,0.0,-0.006114588100848528,0.0,0.0,0.0
+BJ,0.021569600815111167,0.025045532052292208,0.055685686523292866,0.01786680031720378,0.021569600815111167,0.01390442519840307,0.04940761418323669,0.03982178088300037,0.03715504444619189,0.021569600815111167,0.006453239989770987,0.025045532052292208,0.03715504444619189,0.021569600815111167,-0.018597262031625594,0.0,0.0,0.01095505452809764,0.0147194767041825,-0.03582400205750491,0.0,-0.006114588100848528,0.0,0.0,0.0
+EG,0.021569600815111167,0.025045532052292208,0.055685686523292866,0.01786680031720378,0.021569600815111167,0.01390442519840307,0.04940761418323669,0.03982178088300037,0.03715504444619189,0.021569600815111167,0.006453239989770987,0.025045532052292208,0.03715504444619189,0.021569600815111167,-0.018597262031625594,0.0,0.0,0.01095505452809764,0.0147194767041825,-0.03582400205750491,0.0,-0.006114588100848528,0.0,0.0,0.0

From 8eb1c6a9c85592886764f01d3f45b8598951e868 Mon Sep 17 00:00:00 2001
From: "anton.achhammer" <anton.achhammer@oth-regensburg.de>
Date: Wed, 30 Aug 2023 14:03:00 +0200
Subject: [PATCH 48/58] feat: add copy commit rule

---
 Snakefile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Snakefile b/Snakefile
index 10c54642..eee47e69 100644
--- a/Snakefile
+++ b/Snakefile
@@ -367,6 +367,12 @@ rule copy_config:
     script:
         "scripts/copy_config.py"
 
+rule copy_commit:
+    output: SDIR + "/commit_info.txt"
+    shell:
+        """
+        git log -n 1 --pretty=format:"Commit: %H%nAuthor: %an <%ae>%nDate: %ad%nMessage: %s" > {output}
+        """
 
 rule solve_network:
     input:
@@ -377,6 +383,7 @@ rule solve_network:
         + "/prenetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",
         costs=CDIR + "costs_{planning_horizons}.csv",
         configs=SDIR + "/configs/config.yaml",  # included to trigger copy_config rule
+        commit=SDIR + "/commit_info.txt",
     output:
         RDIR
         + "/postnetworks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{sopts}_{planning_horizons}_{discountrate}_{demand}_{h2export}export.nc",

From b591e9f7d5bc2e6f8c26e19e803ccb6fc9286c5f Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 30 Aug 2023 12:05:59 +0000
Subject: [PATCH 49/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 Snakefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Snakefile b/Snakefile
index eee47e69..8ae7b588 100644
--- a/Snakefile
+++ b/Snakefile
@@ -367,13 +367,16 @@ rule copy_config:
     script:
         "scripts/copy_config.py"
 
+
 rule copy_commit:
-    output: SDIR + "/commit_info.txt"
+    output:
+        SDIR + "/commit_info.txt",
     shell:
         """
         git log -n 1 --pretty=format:"Commit: %H%nAuthor: %an <%ae>%nDate: %ad%nMessage: %s" > {output}
         """
 
+
 rule solve_network:
     input:
         overrides="data/override_component_attrs",

From 8ffddb7115aba57e564f39ebd6193f32f6e2b087 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 31 Aug 2023 19:24:29 +0200
Subject: [PATCH 50/58] add the airports workflow

---
 Snakefile                   |   7 ++
 data/airports.csv           | 174 ------------------------------------
 scripts/prepare_airports.py | 116 ++++++++++++++++++++++++
 3 files changed, 123 insertions(+), 174 deletions(-)
 delete mode 100644 data/airports.csv
 create mode 100644 scripts/prepare_airports.py

diff --git a/Snakefile b/Snakefile
index 10c54642..cc6bbc28 100644
--- a/Snakefile
+++ b/Snakefile
@@ -85,6 +85,13 @@ rule prepare_ports:
     script:
         "scripts/prepare_ports.py"
 
+rule prepare_airports:
+    output:
+        ports="data/airports.csv",
+        # TODO move from data to resources
+    script:
+        "scripts/prepare_airports.py"
+
 
 rule prepare_sector_network:
     input:
diff --git a/data/airports.csv b/data/airports.csv
deleted file mode 100644
index e5611ca7..00000000
--- a/data/airports.csv
+++ /dev/null
@@ -1,174 +0,0 @@
-name,country,fraction,y,x,source,disaggregation metric
-Casablanca-Mohammed V,MA,0.428393181,33.373831,-7.579726,,
-Marrakesh,MA,0.265873634,31.60416425,-8.035166526,,
-Agadir,MA,0.083484208,30.325,-9.413056,,
-Fez,MA,0.05893589,33.927222,-4.978056,,
-Tangiers,MA,0.056274782,35.179557,-3.839891,,
-Rabat,MA,0.045757958,34.051389,-6.751389,,
-Nador,MA,0.032104508,34.988889,-3.028333,,
-Oujda,MA,0.029175839,34.787222,-1.923889,,
-Dummy Airport1,NG,0.7,11.423,6.032,,
-Dummy Airport2,NG,0.3,8.34321,9.143,,
-Dummy Airport1,BJ,0.7,10.23,2.32,,
-Dummy Airport2,BJ,0.3,7.3,2.24,,
-Hosea Kutako International Airport,NA,0.341798216,-22.4799,17.4709,http://www.airports.com.na/files/files/Annual%20report.pdf, Disaggregated from total aircraft movement of 14940 in 2015
-Eros Airport,NA,0.398055214,-22.6122,17.0804,http://www.airports.com.na/files/files/Annual%20report.pdf, Disaggregated from total aircraft movement of 17807 in 2015
-Walvis Bay Airport,NA,0.108058567,-22.979,14.6453,http://www.airports.com.na/files/files/Annual%20report.pdf, Disaggregated from total aircraft movement of 4834 in 2015
-Rundu Airport,NA,0.026578741,-17.9565,19.7194,http://www.airports.com.na/files/files/Annual%20report.pdf, Disaggregated from total aircraft movement of 1189 in 2015
-Andimba Toivo ya Toivo Airport,NA,0.054722253,-17.5847,15.9526,http://www.airports.com.na/files/files/Annual%20report.pdf, Disaggregated from total aircraft movement of 2448 in 2015
-Luderitz Airport,NA,0.024969263,-26.687,15.2429,http://www.airports.com.na/files/files/Annual%20report.pdf, Disaggregated from total aircraft movement of 1117 in 2015
-Katima Mulilo Airport,NA,0.03073656,-17.6343,24.1767,http://www.airports.com.na/files/files/Annual%20report.pdf, Disaggregated from total aircraft movement of 1375 in 2015
-Keetmanshoop Airport,NA,0.022912708,-26.5398,18.1114,http://www.airports.com.na/files/files/Annual%20report.pdf, Disaggregated from total aircraft movement of 1025 in 2015
-Arealva,BR,0.00025176979729572385,-22.15705835,-49.07515943787047,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1077726.0 in 2019
-Araraquara,BR,9.970562971090514e-07,-21.812670400000002,-48.13305436466936,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4268.0 in 2019
-Aracaju,BR,0.005775662356044446,-10.98281225,-37.07602527743482,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 24723305.0 in 2019
-Araçatuba,BR,0.00022848241212210187,-21.140815,-50.425139094792286,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 978042.0 in 2019
-Araxá,BR,8.065222961900864e-06,-19.563132000000003,-46.96040575380038,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 34524.0 in 2019
-Belém,BR,0.019787617117580904,-1.3798578,-48.47964610194296,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 84702890.0 in 2019
-Bagé,BR,4.3830295797469594e-06,-31.390997249999998,-54.112838163262815,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 18762.0 in 2019
-Belo Horizonte,BR,7.000349082071392e-05,-19.850977,-43.94919201268867,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 299657.0 in 2019
-Brasília,BR,0.07867501441912658,-15.87174755,-47.91172169662855,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 336776331.0 in 2019
-Boa Vista,BR,0.002269232851945232,2.8467389,-60.69088998008712,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 9713680.0 in 2019
-Barra do Garças,BR,4.1867019995945584e-05,-15.8606725,-52.39006674230171,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 179216.0 in 2019
-Cascavel,BR,0.000695430180776887,-25.000130749999997,-53.501257778932384,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 2976859.0 in 2019
-Cabo Frio,BR,0.00023698495687130082,-22.923360950000003,-42.076586063102596,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1014438.0 in 2019
-Confins,BR,0.03980154615757996,-19.63339785,-43.96836942393943,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 170374531.0 in 2019
-Campo Grande,BR,0.006043392763292088,-20.471576849999998,-54.68224625125994,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 25869352.0 in 2019
-Chapecó,BR,0.0014664287253954426,-27.1342038,-52.6568659851302,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 6277196.0 in 2019
-Parauapebas,BR,0.0009525237634415935,-6.11655575,-49.99774268663904,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4077374.0 in 2019
-Caldas Novas,BR,0.00042815555066974607,-17.7247899,-48.6081742294032,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1832763.0 in 2019
-Campos dos Goytacazes,BR,3.579637685239455e-05,-21.698799450000003,-41.303194297296535,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 153230.0 in 2019
-Corumbá,BR,0.00014961054005704544,-19.0125785,-57.66975905679873,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 640423.0 in 2019
-São José dos Pinhais,BR,0.018143782454919807,-25.53281285,-49.16736187564199,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 77666290.0 in 2019
-Caxias do Sul,BR,0.0007755549143511739,-29.19705975,-51.1861647969846,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 3319841.0 in 2019
-Várzea Grande,BR,0.013474874314588535,-15.651851449999999,-56.11979295012485,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 57680558.0 in 2019
-Cruzeiro do Sul,BR,0.0002551765620522237,-7.598823149999999,-72.77471470435958,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1092309.0 in 2019
-Bonito,BR,0.0001684357011605925,-21.24942965,-56.45190863172819,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 721006.0 in 2019
-Presidente Prudente,BR,0.0007794270343428755,-22.1751078,-51.42479089935418,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 3336416.0 in 2019
-Dourados,BR,0.00029684921680937017,-22.20140635,-54.92640220192091,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1270693.0 in 2019
-Manaus,BR,0.025804147296643383,-3.03553825,-60.04575303314657,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 110457254.0 in 2019
-Feira de Santana,BR,2.396135598364065e-05,-12.2001342,-38.90841809405157,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 102569.0 in 2019
-Foz do Iguaçu,BR,0.008257313987238816,-25.5930616,-54.49060873125,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 35346265.0 in 2019
-Florianópolis,BR,0.011304049143716266,-27.670113450000002,-48.54468648648172,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 48388122.0 in 2019
-Fernando de Noronha,BR,0.0009121147163487074,-3.8546858,-32.4227495764598,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 3904399.0 in 2019
-Fortaleza,BR,0.04172075805124064,-3.7761177999999997,-38.53554028983212,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 178589911.0 in 2019
-Rio de Janeiro,BR,0.04933898278046327,-22.811305750000002,-43.25747587604714,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 211200490.0 in 2019
-Goiânia,BR,0.012699290432341194,-16.634240650000002,-49.2155620047253,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 54360593.0 in 2019
-Gavião Peixoto,BR,3.5100212896118783e-06,-21.77340315,-48.40543622706696,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 15025.0 in 2019
-Guarulhos,BR,0.2577644240074909,-23.43779495,-46.481320528800595,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1103386604.0 in 2019
-Governador Valadares,BR,7.574520817552795e-05,-18.8963614,-41.98338512174777,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 324235.0 in 2019
-Altamira,BR,0.000343358809390238,-3.2537426,-52.252991503644964,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1469782.0 in 2019
-Itaituba,BR,6.550505688042998e-05,-4.2418982,-56.003222084480875,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 280401.0 in 2019
-Ilhéus,BR,0.0025547785153356897,-14.81669225,-39.03309799706171,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 10935987.0 in 2019
-Santana do Paraíso,BR,0.00012840907701952626,-19.4712386,-42.48616461571429,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 549668.0 in 2019
-Imperatriz,BR,0.0016693469691500084,-5.5301267,-47.45800807097166,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 7145808.0 in 2019
-Jaguaruna,BR,0.00047843026891614967,-28.66726125,-49.05437795338313,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 2047969.0 in 2019
-Jundiaí,BR,1.5296918072797724e-06,-23.18119415,-46.942913735809945,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 6548.0 in 2019
-Ji-Paraná,BR,0.00030221832291913055,-10.87032235,-61.84701962387052,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1293676.0 in 2019
-Santa Rita,BR,0.00814002320926997,-7.148930699999999,-34.95030583440195,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 34844190.0 in 2019
-Rio de Janeiro,BR,8.361676673482726e-06,-22.98641695,-43.371449464398225,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 35793.0 in 2019
-Juazeiro do Norte,BR,0.0026808963252132182,-7.219702,-39.267827262760335,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 11475847.0 in 2019
-Joinville,BR,0.001276784319223289,-26.2249688,-48.79862231685878,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 5465404.0 in 2019
-Campina Grande,BR,0.0007584299818629876,-7.27016065,-35.89609699581786,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 3246536.0 in 2019
-Campinas,BR,0.06648764114367078,-23.00605625,-47.1418432690333,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 284607051.0 in 2019
-Lençóis,BR,1.7530249422460923e-05,-12.48182935,-41.277955916935184,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 75040.0 in 2019
-Lages,BR,5.176002376116913e-05,-27.783411,-50.28044200776915,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 221564.0 in 2019
-Londrina,BR,0.002699015977877855,-23.331887549999998,-51.1346212481326,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 11553410.0 in 2019
-Marabá,BR,0.0011675863304401177,-5.3684215,-49.136182375369714,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4997971.0 in 2019
-Almeirim,BR,1.6579448314393013e-06,-0.8896558000000001,-52.60197779839184,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 7097.0 in 2019
-Macaé,BR,1.718660607357446e-05,-22.343171050000002,-41.76577364545359,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 73569.0 in 2019
-Maringá,BR,0.0018267218398281214,-23.4761138,-52.014684538049366,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 7819467.0 in 2019
-Montes Claros,BR,0.0005256283162551197,-16.70518095,-43.82008294490494,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 2250005.0 in 2019
-Marília,BR,0.00011367049178414433,-22.1968134,-49.92775661232895,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 486578.0 in 2019
-Rio Largo,BR,0.012343257233929553,-9.51084465,-35.7925282592017,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 52836557.0 in 2019
-Macapá,BR,0.001870333241119876,0.0505785,-51.07059996039651,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 8006150.0 in 2019
-Mossoró,BR,5.7847253361396537e-05,-5.20144145,-37.36396336093813,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 247621.0 in 2019
-Navegantes,BR,0.004873264733575366,-26.87842885,-48.649354164934465,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 20860501.0 in 2019
-Santo Ângelo,BR,3.984510756588962e-05,-28.28341965,-54.165611958794074,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 170561.0 in 2019
-Parnamirim,BR,2.653365844087302e-06,-5.9114,-35.2477,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 11358.0 in 2019
-Porto Alegre,BR,0.03027737075722751,-29.995268449999998,-51.16639779619565,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 129605338.0 in 2019
-Parnaíba,BR,2.0105121612417803e-05,-2.89181215,-41.72704584737777,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 86062.0 in 2019
-Poços de Caldas,BR,1.0598979428265618e-06,-21.842504849999997,-46.566647194940074,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4537.0 in 2019
-Passo Fundo,BR,0.0006603404804237321,-28.2435244,-52.32669864043906,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 2826654.0 in 2019
-Ponta Grossa,BR,2.8312614322430056e-05,-25.18419055,-50.144196346103016,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 121195.0 in 2019
-Palmas,BR,0.00253639184407977,-10.2918921,-48.3591818921385,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 10857281.0 in 2019
-Pelotas,BR,4.0230567474565805e-05,-31.71980575,-52.32925064938313,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 172211.0 in 2019
-Petrolina,BR,0.0017151982429292705,-9.36148845,-40.57031761357052,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 7342079.0 in 2019
-Pato Branco,BR,2.6294206073362062e-05,-26.2169937,-52.694225841764215,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 112555.0 in 2019
-Porto Seguro,BR,0.0075306052533512845,-16.4389835,-39.08336724314525,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 32235515.0 in 2019
-Porto Velho,BR,0.004547050012862716,-8.70990265,-63.900493918125576,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 19464106.0 in 2019
-Vitória da Conquista,BR,0.0003515319611265682,-14.86450825,-40.862788613186495,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1504768.0 in 2019
-Rio Branco,BR,0.003086085006463708,-9.86366315,-67.88788220300299,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 13210298.0 in 2019
-Recife,BR,0.049499253506309834,-8.1256775,-34.92271835508188,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 211886545.0 in 2019
-Rio de Janeiro,BR,0.029268938019735,-22.91044995,-43.16419567816759,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 125288640.0 in 2019
-Ribeirão Preto,BR,0.0024320987742314864,-21.13330235,-47.77468310119474,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 10410844.0 in 2019
-São Gonçalo do Amarante,BR,0.013542167671833158,-5.77304275,-35.36213529732427,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 57968614.0 in 2019
-São José dos Campos,BR,8.804488344328604e-05,-23.2286857,-45.86031987273927,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 376885.0 in 2019
-São Luís,BR,0.008825505487446155,-2.58899115,-44.23940764493366,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 37778466.0 in 2019
-Santa Maria,BR,4.224406987025065e-05,-29.71297955,-53.68895977371655,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 180830.0 in 2019
-Santarém,BR,0.002194445453239748,-2.42713895,-54.80445559036728,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 9393545.0 in 2019
-Sorriso,BR,7.583608326914918e-05,-12.47872045,-55.67212787844481,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 324624.0 in 2019
-São Paulo,BR,0.07094813298027738,-23.6256872,-46.6612303,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 303700636.0 in 2019
-São José do Rio Preto,BR,0.0021173789352196912,-20.81731545,-49.408467700474546,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 9063654.0 in 2019
-Salvador,BR,0.035940330943500826,-12.91102045,-38.3349721784498,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 153846210.0 in 2019
-Oriximiná,BR,4.0837958845813114e-05,-1.4891128,-56.39509566714774,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 174811.0 in 2019
-Una,BR,0.00015478621537627017,-15.35574585,-38.998548167088046,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 662578.0 in 2019
-Toledo,BR,5.366162597730495e-05,-24.686413199999997,-53.697093609839946,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 229704.0 in 2019
-Teresina,BR,0.005848357525088049,-5.0598744,-42.823510202633514,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 25034484.0 in 2019
-Tefé,BR,0.00015109631279594272,-3.3824045,-64.72460759140549,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 646783.0 in 2019
-Três Lagoas,BR,8.043543762188652e-05,-20.752112099999998,-51.6801087861806,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 344312.0 in 2019
-Tabatinga,BR,0.0003864223903875408,-4.255893,-69.93548245403075,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1654120.0 in 2019
-São Gabriel da Cachoeira,BR,5.027495185846945e-05,-0.1483109,-66.98557275279653,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 215207.0 in 2019
-Paulo Afonso,BR,2.218870762786195e-05,-9.4005831,-38.25002608988869,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 94981.0 in 2019
-Uruguaiana,BR,5.285870130692918e-05,-29.782747,-57.03777922067036,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 226267.0 in 2019
-Uberlândia,BR,0.0032487673096661934,-18.882273650000002,-48.22479831961242,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 13906676.0 in 2019
-Coari,BR,0.0001879640345823912,-4.884467450000001,-65.3564853947976,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 804599.0 in 2019
-Vitória da Conquista,BR,0.0007522637913844981,-14.90817835,-40.91588320783863,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 3220141.0 in 2019
-Varginha,BR,1.9689759359310987e-05,-21.59120195,-45.47401856941612,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 84284.0 in 2019
-Vilhena,BR,9.705506721160821e-05,-12.6940825,-60.09875397255124,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 415454.0 in 2019
-Vitória,BR,0.01223960356030554,-20.256092950000003,-40.28135718192593,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 52392857.0 in 2019
-Goianá,BR,0.0003845184520507463,-21.514398800000002,-43.17076643690476,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1645970.0 in 2019
-São Carlos,BR,9.377492019480685e-05,-21.8730674,-47.90275820931602,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 401413.0 in 2019
-Franca,BR,6.420126794084103e-06,-20.59294565,-47.384190424304535,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 27482.0 in 2019
-Rio Grande,BR,2.0106990508944717e-06,-32.0824308,-52.16620090786992,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 8607.0 in 2019
-Barretos,BR,6.7752171341979165e-06,-20.58423865,-48.59509323988925,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 29002.0 in 2019
-Barreiras,BR,0.0003103968477329756,-12.0794806,-45.009705353357425,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1328685.0 in 2019
-Ubaporanga,BR,4.493995311032526e-06,-19.7240798,-42.11226009626729,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 19237.0 in 2019
-Diamantina,BR,2.7925986353424554e-06,-18.23369525,-43.65036719883837,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 11954.0 in 2019
-Januária,BR,5.723495613676607e-08,-15.473317399999999,-44.38438081278936,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 245.0 in 2019
-Porto de Moz,BR,4.551931103366885e-06,-1.74273455,-52.237087731100814,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 19485.0 in 2019
-Patos de Minas,BR,3.2074936643175434e-06,-18.67291465,-46.4919252194623,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 13730.0 in 2019
-Teixeira de Freitas,BR,8.190789447302953e-05,-17.52360045,-39.67223125277019,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 350615.0 in 2019
-Teófilo Otoni,BR,1.2626732159968187e-05,-17.8926588,-41.51225168213598,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 54050.0 in 2019
-Araçuaí,BR,2.227257435950725e-06,-16.8528571,-42.04755651808042,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 9534.0 in 2019
-Valença,BR,4.828224093664653e-05,-13.29668645,-38.992338482280566,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 206677.0 in 2019
-Viçosa,BR,1.138391596956984e-06,-20.7465531,-42.840199827585536,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4873.0 in 2019
-Breves,BR,4.898377797043718e-06,-1.6351127,-50.44221729198899,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 20968.0 in 2019
-Almeirim,BR,1.3035553275230803e-06,-1.47928445,-52.57808006493803,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 5580.0 in 2019
-Pouso Alegre,BR,1.0183149951027073e-06,-22.2890865,-45.919592421375825,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4359.0 in 2019
-Cornélio Procópio,BR,6.664952239109942e-07,-23.153039800000002,-50.60303981295276,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 2853.0 in 2019
-Cianorte,BR,1.943185163859674e-06,-23.692075850000002,-52.64238296065088,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 8318.0 in 2019
-Francisco Beltrão,BR,1.0517215205213096e-06,-26.05892195,-53.062941314576875,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4502.0 in 2019
-Campo Mourão,BR,2.576974698549659e-06,-24.009449,-52.35696610253834,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 11031.0 in 2019
-Cacoal,BR,0.00024033074887861006,-11.4964046,-61.44992918812463,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 1028760.0 in 2019
-Arapongas,BR,1.6665884778762821e-06,-23.35378915,-51.491387491949524,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 7134.0 in 2019
-Aripuanã,BR,1.684342994881973e-07,-10.1872182,-59.4575804731363,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 721.0 in 2019
-Paranaguá,BR,1.1257765454003089e-06,-25.53969285,-48.53062440956646,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4819.0 in 2019
-Paranavaí,BR,1.8941266300281603e-06,-23.08972335,-52.490171446822714,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 8108.0 in 2019
-São Borja,BR,1.6654204175469605e-06,-28.65273785,-56.031004260587636,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 7129.0 in 2019
-União da Vitória,BR,2.01490406808003e-06,-26.23353875,-51.06653433238267,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 8625.0 in 2019
-Santa Rosa,BR,3.2002516902757486e-06,-27.90718635,-54.52187693431458,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 13699.0 in 2019
-Barcelos,BR,9.42975103861454e-06,-0.9797751,-62.92253406685408,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 40365.0 in 2019
-Carauari,BR,5.146613978231178e-05,-4.87146245,-66.89752105783637,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 220306.0 in 2019
-Eirunepé,BR,2.9121379294452442e-05,-6.63934205,-69.881439864414,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 124657.0 in 2019
-São Félix do Araguaia,BR,4.643273421119847e-06,-11.632334199999999,-50.68919754280353,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 19876.0 in 2019
-Araguaína,BR,0.00010890994510596057,-7.2280456,-48.24099228052751,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 466200.0 in 2019
-Água Boa,BR,1.2963367146878718e-05,-14.01962765,-52.15026599220484,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 55491.0 in 2019
-Juína,BR,5.870904827237012e-06,-11.41979945,-58.70898540544404,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 25131.0 in 2019
-Coari,BR,1.4165301225750805e-05,-4.1331087,-63.13174905775732,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 60636.0 in 2019
-Lábrea,BR,1.3118018334480918e-05,-7.2795064499999995,-64.76944159805764,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 56153.0 in 2019
-Rio Verde,BR,9.896087444492958e-05,-17.834426,-50.9559705443272,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 423612.0 in 2019
-Maués,BR,4.016492248405792e-06,-3.3699246499999997,-57.724955102848156,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 17193.0 in 2019
-Rondonópolis,BR,1.1276454419272237e-06,-16.5834324,-54.72481645658263,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 4827.0 in 2019
-Tangará da Serra,BR,9.250804196162447e-06,-14.66279795,-57.44358442390656,"['https://www.gov.br/anac/pt-br/assuntos/dados-e-estatisticas/dados-estatisticos/arquivos/', 'https://www.anac.gov.br/acesso-a-informacao/dados-abertos/areas-de-atuacao/todos-os-dados-abertos']",Disaggregated from fuel consumption of 39599.0 in 2019
-Dummy Airport3,AE,1,24.4539,54.3773,,
diff --git a/scripts/prepare_airports.py b/scripts/prepare_airports.py
new file mode 100644
index 00000000..65cfdf0d
--- /dev/null
+++ b/scripts/prepare_airports.py
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+import logging
+import os
+from pathlib import Path
+
+import country_converter as coco
+import helpers
+import numpy as np
+import pandas as pd
+
+# from helpers import configure_logging
+
+
+# logger = logging.getLogger(__name__)
+
+
+def download_airports():
+    """
+    Downloads the world airports as .csv File in addition to runnways information.
+    The following csv file was downloaded from the webpage https://ourairports.com/data/ as a .csv file. The dataset contains 74844 airports.
+    """
+    fn = "https://davidmegginson.github.io/ourairports-data/airports.csv"
+    storage_options = {"User-Agent": "Mozilla/5.0"}
+    airports_csv = pd.read_csv(
+        fn, index_col=0, storage_options=storage_options, encoding="utf8"
+    )
+
+    fn = "https://davidmegginson.github.io/ourairports-data/runways.csv"
+    storage_options = {"User-Agent": "Mozilla/5.0"}
+    runways_csv = pd.read_csv(
+        fn, index_col=0, storage_options=storage_options, encoding="utf8"
+    )
+
+    return (airports_csv, runways_csv)
+
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from helpers import mock_snakemake, sets_path_to_root
+
+        os.chdir(os.path.dirname(os.path.abspath(__file__)))
+        snakemake = mock_snakemake("prepare_airports")
+        sets_path_to_root("pypsa-earth-sec")
+    # configure_logging(snakemake)
+
+    # run = snakemake.config.get("run", {})
+    # RDIR = run["name"] + "/" if run.get("name") else ""
+    # store_path_data = Path.joinpath(Path().cwd(), "data")
+    # country_list = country_list_to_geofk(snakemake.config["countries"])'
+
+    # Prepare downloaded data
+    airports_csv = download_airports()[0].copy()
+    airports_csv = airports_csv[
+        [
+            "ident",
+            "type",
+            "name",
+            "latitude_deg",
+            "longitude_deg",
+            "elevation_ft",
+            "continent",
+            "iso_country",
+            "iso_region",
+            "municipality",
+            "scheduled_service",
+            "iata_code",
+        ]
+    ]
+    airports_csv.loc[airports_csv["iso_country"].isnull(), "iso_country"] = "NA"
+    airports_csv = airports_csv.rename(columns={"latitude_deg": "y"})
+    airports_csv = airports_csv.rename(columns={"longitude_deg": "x"})
+
+    runways_csv = download_airports()[1].copy()
+    runways_csv = runways_csv[
+        ["airport_ident", "length_ft", "width_ft", "surface", "lighted", "closed"]
+    ]
+    runways_csv = runways_csv.drop_duplicates(subset=["airport_ident"])
+
+    airports_original = pd.merge(
+        airports_csv, runways_csv, how="left", left_on="ident", right_on="airport_ident"
+    )
+    airports_original = airports_original.drop("airport_ident", axis=1)
+
+    df = airports_original.copy()
+
+    # Keep only airports that are of type medium and large
+    #df = df.loc[df["type"].isin(["large_airport", "medium_airport"])]
+
+    #Filtering out the military airbases and keeping only commercial airports
+    df = df[~df.iata_code.isnull()]
+
+    # Keep only airports that have schedules
+    df = df.loc[df["scheduled_service"].isin(["yes"])]
+
+    df.insert(2, "airport_size_nr", 1)
+    df.loc[df["type"].isin(["medium_airport"]), "airport_size_nr"] = 1
+    df.loc[df["type"].isin(["large_airport"]), "airport_size_nr"] = 3
+
+    # Calculate the number of total airports size
+    df1 = df.copy()
+    df1 = df1.groupby(["iso_country"]).sum("airport_size_nr")
+    df1 = df1[["airport_size_nr"]]
+    df1 = df1.rename(columns={"airport_size_nr": "Total_airport_size_nr"}).reset_index()
+
+    # Merge dataframes to get additional info on runnway for most ports
+    airports = pd.merge(
+        df, df1, how="left", left_on="iso_country", right_on="iso_country"
+    )
+
+    # Calculate fraction based on size
+    airports["fraction"] = (
+        airports["airport_size_nr"] / airports["Total_airport_size_nr"]
+    )
+
+    # Save
+    airports.to_csv(snakemake.output[0], sep=",", encoding="utf-8", header="true")

From 79a61baff885e26e7eb0153e34fa69f6342a4d47 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 31 Aug 2023 17:25:21 +0000
Subject: [PATCH 51/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 Snakefile                   | 1 +
 scripts/prepare_airports.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Snakefile b/Snakefile
index cc6bbc28..ff422fe4 100644
--- a/Snakefile
+++ b/Snakefile
@@ -85,6 +85,7 @@ rule prepare_ports:
     script:
         "scripts/prepare_ports.py"
 
+
 rule prepare_airports:
     output:
         ports="data/airports.csv",
diff --git a/scripts/prepare_airports.py b/scripts/prepare_airports.py
index 65cfdf0d..e53cc0cb 100644
--- a/scripts/prepare_airports.py
+++ b/scripts/prepare_airports.py
@@ -84,9 +84,9 @@ def download_airports():
     df = airports_original.copy()
 
     # Keep only airports that are of type medium and large
-    #df = df.loc[df["type"].isin(["large_airport", "medium_airport"])]
+    # df = df.loc[df["type"].isin(["large_airport", "medium_airport"])]
 
-    #Filtering out the military airbases and keeping only commercial airports
+    # Filtering out the military airbases and keeping only commercial airports
     df = df[~df.iata_code.isnull()]
 
     # Keep only airports that have schedules

From 727a7827bb77e07805f34f7a99de4c25b66915f1 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Thu, 31 Aug 2023 19:55:40 +0200
Subject: [PATCH 52/58] renaming country column

---
 scripts/prepare_airports.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/prepare_airports.py b/scripts/prepare_airports.py
index e53cc0cb..358784c7 100644
--- a/scripts/prepare_airports.py
+++ b/scripts/prepare_airports.py
@@ -112,5 +112,8 @@ def download_airports():
         airports["airport_size_nr"] / airports["Total_airport_size_nr"]
     )
 
+    # Rename columns
+    airports = airports.rename(columns={"iso_country":"country"})
+
     # Save
     airports.to_csv(snakemake.output[0], sep=",", encoding="utf-8", header="true")

From 8ee092bf479041ef6dfeece2e3333e221c0a9806 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 31 Aug 2023 17:56:48 +0000
Subject: [PATCH 53/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_airports.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prepare_airports.py b/scripts/prepare_airports.py
index 358784c7..7b9fc1cd 100644
--- a/scripts/prepare_airports.py
+++ b/scripts/prepare_airports.py
@@ -113,7 +113,7 @@ def download_airports():
     )
 
     # Rename columns
-    airports = airports.rename(columns={"iso_country":"country"})
+    airports = airports.rename(columns={"iso_country": "country"})
 
     # Save
     airports.to_csv(snakemake.output[0], sep=",", encoding="utf-8", header="true")

From b2f55b09477a4fb5ffd61b7f72689a6e17b12c44 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Fri, 1 Sep 2023 11:32:32 +0200
Subject: [PATCH 54/58] filter out small airports and add sizing factor to
 config

---
 config.default.yaml         | 1 +
 scripts/prepare_airports.py | 4 ++--
 test/config.test1.yaml      | 1 +
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index a7bea787..b158df07 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -263,6 +263,7 @@ sector:
   SMR: true
   cc_fraction: 0.9
   cc: true
+  airport_sizing_factor: 3
 
   conventional_generation: # generator : carrier
     OCGT: gas
diff --git a/scripts/prepare_airports.py b/scripts/prepare_airports.py
index 7b9fc1cd..df5e36c5 100644
--- a/scripts/prepare_airports.py
+++ b/scripts/prepare_airports.py
@@ -84,7 +84,7 @@ def download_airports():
     df = airports_original.copy()
 
     # Keep only airports that are of type medium and large
-    # df = df.loc[df["type"].isin(["large_airport", "medium_airport"])]
+    df = df.loc[df["type"].isin(["large_airport", "medium_airport"])]
 
     # Filtering out the military airbases and keeping only commercial airports
     df = df[~df.iata_code.isnull()]
@@ -94,7 +94,7 @@ def download_airports():
 
     df.insert(2, "airport_size_nr", 1)
     df.loc[df["type"].isin(["medium_airport"]), "airport_size_nr"] = 1
-    df.loc[df["type"].isin(["large_airport"]), "airport_size_nr"] = 3
+    df.loc[df["type"].isin(["large_airport"]), "airport_size_nr"] = snakemake.config["sector"]["airport_sizing_factor"]
 
     # Calculate the number of total airports size
     df1 = df.copy()
diff --git a/test/config.test1.yaml b/test/config.test1.yaml
index c0ad2928..cff0468d 100644
--- a/test/config.test1.yaml
+++ b/test/config.test1.yaml
@@ -265,6 +265,7 @@ sector:
   SMR: true
   cc_fraction: 0.9
   cc: true
+  airport_sizing_factor: 3
 
   conventional_generation: # generator : carrier
     OCGT: gas

From 2cebbaddf8b8f34c1bf1a8f3bf28891d0e982f9e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 1 Sep 2023 11:37:25 +0000
Subject: [PATCH 55/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_airports.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/prepare_airports.py b/scripts/prepare_airports.py
index df5e36c5..d8ff462c 100644
--- a/scripts/prepare_airports.py
+++ b/scripts/prepare_airports.py
@@ -94,7 +94,9 @@ def download_airports():
 
     df.insert(2, "airport_size_nr", 1)
     df.loc[df["type"].isin(["medium_airport"]), "airport_size_nr"] = 1
-    df.loc[df["type"].isin(["large_airport"]), "airport_size_nr"] = snakemake.config["sector"]["airport_sizing_factor"]
+    df.loc[df["type"].isin(["large_airport"]), "airport_size_nr"] = snakemake.config[
+        "sector"
+    ]["airport_sizing_factor"]
 
     # Calculate the number of total airports size
     df1 = df.copy()

From d2ef6213871347d2ffdbb9bab156859bf979d251 Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Wed, 6 Sep 2023 12:52:45 +0200
Subject: [PATCH 56/58] fix minor bug

---
 scripts/prepare_energy_totals.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index c4ba94bc..e2ed866d 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -81,7 +81,7 @@ def calculate_end_values(df):
         options["shipping_hydrogen_share"], demand_sc + "_" + str(investment_year)
     )
 
-    energy_totals = base_energy_totals * efficiency_gains * growth_factors
+    energy_totals = base_energy_totals * efficiency_gains.loc[countries] * growth_factors.loc[countries]
 
     # Residential
     efficiency_heat_oil_to_elec = snakemake.config["sector"][
@@ -270,4 +270,4 @@ def calculate_end_values(df):
 
     energy_totals = energy_totals.dropna(axis=1, how="all")
 
-    energy_totals.to_csv(snakemake.output.energy_totals)
+    energy_totals.fillna(0).to_csv(snakemake.output.energy_totals)

From c5a08bb19208d53fc04b712df7dcbe2b77b0395c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 12 Sep 2023 10:35:57 +0000
Subject: [PATCH 57/58] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 scripts/prepare_energy_totals.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/prepare_energy_totals.py b/scripts/prepare_energy_totals.py
index e2ed866d..a6727c5b 100644
--- a/scripts/prepare_energy_totals.py
+++ b/scripts/prepare_energy_totals.py
@@ -81,7 +81,11 @@ def calculate_end_values(df):
         options["shipping_hydrogen_share"], demand_sc + "_" + str(investment_year)
     )
 
-    energy_totals = base_energy_totals * efficiency_gains.loc[countries] * growth_factors.loc[countries]
+    energy_totals = (
+        base_energy_totals
+        * efficiency_gains.loc[countries]
+        * growth_factors.loc[countries]
+    )
 
     # Residential
     efficiency_heat_oil_to_elec = snakemake.config["sector"][

From 97c2f2f0a6c0487a83a0a9d9a35e5d456f680a0c Mon Sep 17 00:00:00 2001
From: Hazem-IEG <hazem.abdel-khalek@ieg.fraunhofer.de>
Date: Tue, 12 Sep 2023 12:41:43 +0200
Subject: [PATCH 58/58] add default values for EG and ignore slurm folder

---
 .gitignore                       | 2 +-
 data/demand/district_heating.csv | 1 +
 data/demand/fuel_shares.csv      | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 0ea1d517..ccf9972e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,7 +54,7 @@ doc/_build
 /sample_pypsa_eur
 
 /notebooks
-
+/slurm
 # File extensions
 *.org
 *.nc
diff --git a/data/demand/district_heating.csv b/data/demand/district_heating.csv
index 122a5a1d..e50c6257 100644
--- a/data/demand/district_heating.csv
+++ b/data/demand/district_heating.csv
@@ -2,3 +2,4 @@ country,current,potential
 MA, 0, 0
 NG, 0, 0
 BJ, 0, 0
+EG, 0, 0
diff --git a/data/demand/fuel_shares.csv b/data/demand/fuel_shares.csv
index a380b12e..db7ee949 100644
--- a/data/demand/fuel_shares.csv
+++ b/data/demand/fuel_shares.csv
@@ -2,3 +2,4 @@ country,oil residential heat share,biomass residential heat share,biomass to ele
 MA, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
 NG, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
 BJ, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75
+EG, 0.6667, 0.75,0.0, 0.5, 0.0, 0.5,0.6,0,0,0.75