Fix methanol final energy allocation bug

Author: macflo8

message_ix_models/data/material/reporting/fe_reporting_aggregates.yaml

@@ -95,30 +95,39 @@ level_2:
         fe_pe_chem_other_h2 ]
 
 level_1:
+
+  # some Chemicals|<Fuels>|<Sub-fuel> variables are not specified here
+  #   because they are generated later in the workflow, after the
+  #   Methanol/HVC/Ammonia subcomponents are generated.
+  # The function run_reporting.add_chemicals_to_final_energy_variables
+  #   does this aggregation and updates all related aggregates.
+  # Only the ones not generated there are mapped here
+
+  Chemicals|Liquids|Other:
+    short: fe_pe_chem_oth
+    components:
+      [ fe_pe_hvc_oth ]
+  Chemicals|Liquids|Biomass:
+    short: fe_pe_chem_oth
+    components:
+      [ fe_pe_hvc_bio_eth ]
+
   Chemicals|High-Value Chemicals|Electricity:
     short: fe_pe_hvc_el
     components:
       [fe_pe_hvc_el_ht, fe_pe_hvc_el_mto, fe_pe_hvc_el_sc]
-  Chemicals|High-Value Chemicals|Liquids|Oil:
-    short: fe_pe_hvc_oil
-    components:
-      [ fe_pe_hvc_oil_l, fe_pe_hvc_oil_h ]
   Chemicals|High-Value Chemicals|Liquids:
     short: fe_pe_hvc_liq
     components:
       [ fe_pe_hvc_oil_l, fe_pe_hvc_oil_h, fe_pe_hvc_bio_eth, fe_pe_hvc_oth]
+  Chemicals|High-Value Chemicals|Liquids|Oil:
+    short: fe_pe_hvc_oil
+    components:
+      [ fe_pe_hvc_oil_l, fe_pe_hvc_oil_h ]
   Chemicals|High-Value Chemicals|Solids:
     short: fe_pe_hvc_solids
     components:
       [ fe_pe_hvc_bio, fe_pe_hvc_coal, fe_pe_hvc_coke]
-  Chemicals|Liquids|Other:
-    short: fe_pe_chem_oth
-    components:
-      [ fe_pe_hvc_oth ]
-  Chemicals|Liquids|Gases:
-    short: fe_pe_hvc_gases
-    components:
-      [ fe_pe_hvc_gas ]
 
   Chemicals|Other Sector|Electricity:
     short: fe_pe_chem_other_el

message_ix_models/model/material/report/reporter_utils.py

@@ -106,18 +106,33 @@ def add_methanol_share_calculations(rep: message_ix.Reporter, mode: str = "feeds
     if mode == "fuel":
         t_filter2.update({"l": ["primary"]})
 
-    rep.add("out::methanol", "select", "out:nl-t-ya-m-c-l", t_filter2)
-    rep.add("out::methanol-by-tec", "group_sum", "out::methanol", group="t", sum="c")
+    rep.add("out::methanol-prod", "select", "out:nl-t-ya-m-c-l", t_filter2)
     rep.add(
-        "out::methanol-total", "group_sum", "out::methanol", group=["nl", "ya"], sum="t"
+        "out::methanol-prod-by-tec",
+        "group_sum",
+        "out::methanol-prod",
+        group="t",
+        sum="c",
+    )
+    rep.add(
+        "out::methanol-prod-total",
+        "group_sum",
+        "out::methanol-prod",
+        group=["nl", "ya"],
+        sum="t",
+    )
+    rep.add(
+        "share::methanol-prod-by-tec",
+        "div",
+        "out::methanol-prod-by-tec",
+        "out::methanol-prod-total",
     )
-    rep.add("share::methanol", "div", "out::methanol-by-tec", "out::methanol-total")
     for comm, tecs in comm_tec_map.items():
         rep.add(
-            f"share::{comm}methanol",
+            f"share::{comm}-methanol-prod",
             "aggregate",
-            "share::methanol",
-            groups={"t": {f"{comm}methanol": tecs}},
+            "share::methanol-prod-by-tec",
+            groups={"t": {f"{comm}-methanol": tecs}},
             keep=False,
         )
 
@@ -136,10 +151,8 @@ def add_meth_export_calculations(rep: message_ix.Reporter, mode: str = "feedstoc
     ----------
     rep
     """
+    add_methanol_share_calculations(rep, mode=mode)
     for comm in comm_tec_map.keys():
-        if f"share::{comm}methanol" not in rep.keys():
-            add_methanol_share_calculations(rep, mode=mode)
-
         t_filter2 = {"t": "meth_exp", "m": mode}
         rep.add("out::methanol-export", "select", "out:nl-t-ya-m", t_filter2)
         rep.add(
@@ -153,7 +166,7 @@ def add_meth_export_calculations(rep: message_ix.Reporter, mode: str = "feedstoc
             f"out::{comm}methanol-export",
             "mul",
             "out::methanol-export",
-            f"share::{comm}methanol",
+            f"share::{comm}-methanol-prod",
         )
 
 
@@ -174,10 +187,9 @@ def add_meth_import_calculations(rep: message_ix.Reporter, mode: str = "feedstoc
     ----------
     rep
     """
-    for comm in comm_tec_map.keys():
-        if "out::{comm}methanol-export" not in rep.keys():
-            add_meth_export_calculations(rep, mode=mode)
 
+    add_meth_export_calculations(rep, mode=mode)
+    for comm in comm_tec_map.keys():
         t_filter2 = {"t": "meth_imp", "m": mode}
         rep.add("out::methanol-import", "select", "out:nl-t-ya-m", t_filter2)
 
@@ -205,7 +217,7 @@ def add_meth_import_calculations(rep: message_ix.Reporter, mode: str = "feedstoc
             f"share::{comm}methanol-import",
             "div",
             f"out::{comm}methanol-import",
-            "out::methanol-total",
+            "out::methanol-prod-total",
         )
 
 
@@ -220,17 +232,27 @@ def add_biometh_final_share(rep: message_ix.Reporter, mode: str = "feedstock"):
     ----------
     rep
     """
+    add_meth_import_calculations(rep, mode=mode)
+    if mode == "feedstock":
+        t_filter2 = {
+            "t": ["meth_t_d"],
+            "m": [mode],
+        }
+    else:
+        t_filter2 = {
+            "t": ["meth_t_d", "furnace_methanol_refining"],
+            "m": [mode, "high_temp"],
+        }
     for comm in comm_tec_map.keys():
-        if f"out::{comm}methanol-import" not in rep.keys():
-            add_meth_import_calculations(rep, mode=mode)
-
-        t_filter2 = {"t": "meth_t_d", "m": mode}
-        rep.add("in::methanol-final", "select", "in:nl-t-ya-m", t_filter2)
+        rep.add("in::methanol-final0", "select", "in:nl-t-ya-m", t_filter2)
+        rep.add(
+            "in::methanol-final", "sum", "in::methanol-final0", dimensions=["t", "m"]
+        )
         rep.add(
             f"out::{comm}methanol-prod",
             "mul",
-            "out::methanol-total",
-            f"share::{comm}methanol",
+            "out::methanol-prod-total",
+            f"share::{comm}-methanol-prod",
         )
         rep.add(
             f"out::{comm}methanol-final",

message_ix_models/model/material/report/run_reporting.py

@@ -266,7 +266,7 @@ def run_fe_reporting(rep: message_ix.Reporter, model: str, scenario: str):
 
     py_df_all = add_chemicals_to_final_energy_variables(dfs, rep, model, scenario)
 
-    py_df_all = update_liquids_subvariables(rep, py_df_all, model, scenario)
+    py_df_all = split_fe_other(rep, py_df_all, model, scenario)
     df_final = (
         py_df_all.filter(unit="dimensionless", keep=False)
         .convert_unit("GWa", "EJ")
@@ -340,13 +340,13 @@ def add_chemicals_to_final_energy_variables(
     updated_fe_totals = []
     for comm in [
         "Electricity",
+        "Gases",
+        "Gases|Gas",
         "Liquids",
         "Liquids|Oil",
         "Solids",
         "Solids|Biomass",
         "Solids|Coal",
-        "Gases",
-        "Gases|Gas",
         "Hydrogen",
     ]:
         updated_fe_totals.append(
@@ -363,10 +363,36 @@ def add_chemicals_to_final_energy_variables(
     return py_df_all
 
 
-def update_liquids_subvariables(
+def split_fe_other(
     rep: message_ix.Reporter, py_df_all: pyam.IamDataFrame, model: str, scenario: str
 ):
+    """This function takes the Final Energy|Industry|*|Liquids|Other values
+    and reallocates it to Liquids|Biomass/Coal/Oil/Gas based on the methanol
+    feedstock shares.
+    1) calculates the feedstock shares of methanol production with message_ix.Reporter
+    2) append the shares as temporary iamc variables them to the existing reporting
+        pyam object
+    3) Uses pyam multiply feature to calculate shares with each "Liquids|Other"
+        timeseries
+    4) Uses pyam aggregate to sum existing Liquids|Biomass/Coal/Oil/Gas with new
+        variables and store in separate pyam object
+    5) Filters out existing (outdated) Liquids|Biomass/Coal/Oil/Gas from reporting
+        pyam object
+    6) Concats the updated variables with the full reporting
+
+    Parameters
+    ----------
+    rep
+    py_df_all
+    model
+    scenario
+
+    Returns
+    -------
+
+    """
     add_biometh_final_share(rep, mode="fuel")
+    # set temporary filter on Reporter to speed up queries
     rep.set_filters(
         t=[
             "meth_bunker",
@@ -385,6 +411,7 @@ def update_liquids_subvariables(
             "meth_exp",
             "meth_imp",
             "meth_ind_fs",
+            "furnace_methanol_refining"
         ]
     )
     for c, full_name in zip(