Deflate wholesale energy prices

etl/steps/data/garden/ember/2025-02-03/european_wholesale_electricity_prices.py

@@ -18,12 +18,14 @@ def run(dest_dir: str) -> None:
     #
     # Load inputs.
     #
-    # Load meadow dataset.
+    # Load meadow dataset, and read its main table.
     ds_meadow = paths.load_dataset("european_wholesale_electricity_prices")
-
-    # Read table from meadow dataset.
     tb_monthly = ds_meadow.read("european_wholesale_electricity_prices")
 
+    # Load Eurostat Producer Prices in Industry dataset, and read its main table.
+    ds_ppi = paths.load_dataset("producer_prices_in_industry")
+    tb_ppi = ds_ppi.read("producer_prices_in_industry")
+
     #
     # Process data.
     #
@@ -33,6 +35,20 @@ def run(dest_dir: str) -> None:
     # Harmonize country names.
     tb_monthly = geo.harmonize_countries(df=tb_monthly, countries_file=paths.country_mapping_path)
 
+    # Select Producer Prices Index for classification "[MIG_NRG] MIG - energy".
+    tb_ppi = tb_ppi[tb_ppi["classification"] == "MIG_NRG"].drop(columns=["classification"]).reset_index(drop=True)
+
+    # Adapt dates in PPI dataset to match the monthly electricity prices.
+    tb_ppi["date"] = tb_ppi["date"].str.strip() + "-01"
+    assert tb_ppi["date"].str.len().eq(10).all(), "Unexpected date format in PPI dataset."
+
+    # Combine energy prices table with PPI table.
+    tb_monthly = tb_monthly.merge(tb_ppi, on=["country", "date"], how="left")
+
+    # Adjust monthly prices for inflation.
+    # NOTE: When doing this, many prices will be lost (e.g. UK data from 2020 onwards).
+    tb_monthly["price"] = tb_monthly["price"] * 100 / tb_monthly["ppi"]
+
     # Ember provides monthly data, so we can create a monthly table of wholesale electricity prices.
     # But we also need to create an annual table of average wholesale electricity prices.
     tb_annual = tb_monthly.copy()

etl/steps/data/garden/eurostat/2025-02-03/gas_and_electricity_prices.py

@@ -877,15 +877,11 @@ def run(dest_dir: str) -> None:
     # As recommended by Eurostat, we will use only the following:
     # [TOT_X_NRG] Overall index excluding energy
     # Excluding the energy component helps in depicting the prices adjusted for the underlying inflationary trends, without being distorted by the volatility typically associated with energy commodities. In this respect, adjusted prices will include the impact of the energy cost changes.
-    tb_hicp = tb_hicp[tb_hicp["coicop"] == "TOT_X_NRG"].reset_index(drop=True)
+    tb_hicp = tb_hicp[tb_hicp["classification"] == "TOT_X_NRG"].reset_index(drop=True)
 
     # Calculate average HICP for each country and year.
     tb_hicp["year"] = tb_hicp["date"].str[0:4].astype(int)
-    tb_hicp = (
-        tb_hicp.groupby(["country", "year"], observed=True, as_index=False)
-        .agg({"value": "mean"})
-        .rename(columns={"value": "hicp"}, errors="raise")
-    )
+    tb_hicp = tb_hicp.groupby(["country", "year"], observed=True, as_index=False).agg({"hicp": "mean"})
 
     # Add HICP column to main table.
     tb = tb.merge(tb_hicp, how="left", on=["country", "year"])

etl/steps/data/garden/eurostat/2025-02-03/harmonised_index_of_consumer_prices.meta.yml

@@ -12,7 +12,7 @@ dataset:
 tables:
   harmonised_index_of_consumer_prices:
     variables:
-      value:
+      hicp:
         title: Index value
         unit: ""
         short_unit: ""

etl/steps/data/garden/eurostat/2025-02-03/harmonised_index_of_consumer_prices.py

@@ -10,10 +10,10 @@
 
 # Columns to keep and how to rename them.
 COLUMNS = {
-    "coicop": "coicop",
+    "coicop": "classification",
     "geo": "country",
     "time": "date",
-    "value": "value",
+    "value": "hicp",
 }
 
 # NOTE: The description of the flags is given below the main table in
@@ -25,8 +25,8 @@
 
 
 def sanity_check_outputs(tb: Table) -> None:
-    assert tb["value"].notnull().all(), "Some values are missing."
-    assert (tb["value"] >= 0).all(), "Negative values are not allowed."
+    assert tb["hicp"].notnull().all(), "Some values are missing."
+    assert (tb["hicp"] >= 0).all(), "Negative values are not allowed."
 
 
 def run(dest_dir: str) -> None:
@@ -49,15 +49,15 @@ def run(dest_dir: str) -> None:
     )
 
     # Separate flags from values.
-    tb["flag"] = tb["value"].astype("string").str.extract(r"([a-z]+)", expand=False)
+    tb["flag"] = tb["hicp"].astype("string").str.extract(r"([a-z]+)", expand=False)
     tb["flag"] = tb["flag"].map(FLAGS).fillna("")
-    tb["value"] = tb["value"].str.replace(r"[a-z]", "", regex=True).str.strip().astype("Float64")
+    tb["hicp"] = tb["hicp"].str.replace(r"[a-z]", "", regex=True).str.strip().astype("Float64")
 
     # Run sanity checks on outputs.
     sanity_check_outputs(tb=tb)
 
     # Improve main table format.
-    tb = tb.format(keys=["country", "date", "coicop"])
+    tb = tb.format(keys=["country", "date", "classification"])
 
     #
     # Save outputs.

etl/steps/data/garden/eurostat/2025-02-03/producer_prices_in_industry.meta.yml

@@ -12,7 +12,7 @@ dataset:
 tables:
   producer_prices_in_industry:
     variables:
-      value:
+      ppi:
         title: Index value
         unit: ""
         short_unit: ""

etl/steps/data/garden/eurostat/2025-02-03/producer_prices_in_industry.py

@@ -14,7 +14,7 @@
     "nace_r2": "classification",
     "geo": "country",
     "time": "date",
-    "value": "value",
+    "value": "ppi",
 }
 
 # NOTE: The description of the flags is given below the main table in
@@ -32,8 +32,8 @@
 
 
 def sanity_check_outputs(tb: Table) -> None:
-    assert tb["value"].notnull().all(), "Some values are missing."
-    assert (tb["value"] >= 0).all(), "Negative values are not allowed."
+    assert tb["ppi"].notnull().all(), "Some values are missing."
+    assert (tb["ppi"] >= 0).all(), "Negative values are not allowed."
 
 
 def run(dest_dir: str) -> None:
@@ -56,9 +56,9 @@ def run(dest_dir: str) -> None:
     )
 
     # Separate flags from values.
-    tb["flag"] = tb["value"].astype("string").str.extract(r"([a-z]+)", expand=False)
+    tb["flag"] = tb["ppi"].astype("string").str.extract(r"([a-z]+)", expand=False)
     tb["flag"] = tb["flag"].map(FLAGS).fillna("")
-    tb["value"] = tb["value"].str.replace(r"[a-z]", "", regex=True).str.strip().astype("Float64")
+    tb["ppi"] = tb["ppi"].str.replace(r"[a-z]", "", regex=True).str.strip().astype("Float64")
 
     # Run sanity checks on outputs.
     sanity_check_outputs(tb=tb)