Merge pull request #295 from iiasa/issue/294

Confirm support for minimum versions

Author: khaeru

.github/workflows/pytest.yaml

@@ -74,7 +74,8 @@ jobs:
         # - Versions of ixmp and message_ix to test.
         # - Latest supported Python version for those or other dependencies.
         # Minimum version given in pyproject.toml + earlier version of Python
-        - { upstream: v3.6.0, python: "3.11" }  # 2022-08-18
+        # For this job only, the oldest version of Python supported by message-ix-models
+        - { upstream: v3.6.0, python: "3.9" }  # Released 2022-08-18
         - { upstream: v3.7.0, python: "3.11" }  # 2023-05-17
         - { upstream: v3.8.0, python: "3.12" }  # 2024-01-12
         # Latest released version + latest released Python
@@ -133,6 +134,7 @@ jobs:
         v, result = "${{ matrix.version.upstream }}".replace("main", "vmain"), []
         for condition, dependency in (
             (v <= "v3.6.0", "dask < 2024.3.0"),  # dask[dataframe] >= 2024.3.0 requires dask-expr and in turn pandas >= 2.0 (#156)
+            (v <= "v3.6.0", "numpy < 2.0"),
             (v <= "v3.6.0", "pandas < 2.0"),
             (v >= "v3.7.0", "dask[dataframe] < 2024.11.0"),  # dask >= 2024.11.0 changes handling of dict (will be addressed in #225)
             (v <= "v3.7.0", "genno < 1.25"),  # Upstream versions < 3.8.0 import genno.computations, removed in 1.25.0 (#156)

message_ix_models/model/material/build.py

@@ -1,6 +1,6 @@
 import logging
 from collections.abc import Mapping
-from typing import Any
+from typing import Any, Optional
 
 import message_ix
 import pandas as pd
@@ -90,7 +90,7 @@ def build(
     scenario: message_ix.Scenario,
     old_calib: bool,
     modify_existing_constraints: bool = True,
-    iea_data_path: str | None = None,
+    iea_data_path: Optional[str] = None,
 ) -> message_ix.Scenario:
     """Set up materials accounting on `scenario`."""
     node_suffix = context.model.regions

message_ix_models/model/material/data_petro.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from typing import Union
 
 import message_ix
 import pandas as pd
@@ -233,7 +234,7 @@ def assign_input_outpt(
     split: str,
     param_name: str,
     regions: pd.DataFrame,
-    val: float | int,
+    val: Union[float, int],
     t: str,
     rg: str,
     global_region: str,

message_ix_models/model/material/data_util.py

@@ -1,7 +1,7 @@
 import os
 from collections.abc import Mapping
 from functools import lru_cache
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING, Literal, Optional
 
 import ixmp
 import message_ix
@@ -1013,7 +1013,7 @@ def read_iea_tec_map(tec_map_fname: str) -> pd.DataFrame:
 
 
 def get_hist_act_data(
-    map_fname: str, years: list or None = None, iea_data_path: str | None = None
+    map_fname: str, years: list or None = None, iea_data_path: Optional[str] = None
 ) -> pd.DataFrame:
     """
     reads IEA DB, maps and aggregates variables to MESSAGE technologies

message_ix_models/model/material/material_demand/material_demand_calc.py

@@ -1,6 +1,6 @@
 import logging
 from pathlib import Path
-from typing import Literal
+from typing import Literal, Union
 
 import message_ix
 import numpy as np
@@ -57,12 +57,12 @@
 log = logging.getLogger(__name__)
 
 
-def steel_function(x: pd.DataFrame | float, a: float, b: float, m: float):
+def steel_function(x: Union[pd.DataFrame, float], a: float, b: float, m: float):
     gdp_pcap, del_t = x
     return a * np.exp(b / gdp_pcap) * (1 - m) ** del_t
 
 
-def cement_function(x: pd.DataFrame | float, a: float, b: float):
+def cement_function(x: Union[pd.DataFrame, float], a: float, b: float):
     gdp_pcap = x[0]
     return a * np.exp(b / gdp_pcap)
 
@@ -92,12 +92,14 @@ def cement_function(x: pd.DataFrame | float, a: float, b: float):
 }
 
 
-def gompertz(phi: float, mu: float, y: pd.DataFrame | float, baseyear: int = 2020):
+def gompertz(
+    phi: float, mu: float, y: Union[pd.DataFrame, float], baseyear: int = 2020
+):
     return 1 - np.exp(-phi * np.exp(-mu * (y - baseyear)))
 
 
 def read_timer_pop(
-    datapath: str | Path, material: Literal["cement", "steel", "aluminum"]
+    datapath: Union[str, Path], material: Literal["cement", "steel", "aluminum"]
 ):
     df_population = pd.read_excel(
         f"{datapath}/{material_data[material]['dir']}{material_data[material]['file']}",
@@ -115,7 +117,7 @@ def read_timer_pop(
 
 
 def read_timer_gdp(
-    datapath: str | Path, material: Literal["cement", "steel", "aluminum"]
+    datapath: Union[str, Path], material: Literal["cement", "steel", "aluminum"]
 ):
     # Read GDP per capita data
     df_gdp = pd.read_excel(
@@ -165,7 +167,7 @@ def project_demand(df: pd.DataFrame, phi: float, mu: float):
     return df_demand[["region", "year", "demand_tot"]]
 
 
-def read_base_demand(filepath: str | Path):
+def read_base_demand(filepath: Union[str, Path]):
     with open(filepath, "r") as file:
         yaml_data = file.read()
 

message_ix_models/model/transport/base.py

@@ -1,7 +1,7 @@
 """Data preparation for the MESSAGEix-GLOBIOM base model."""
 
 from functools import partial
-from itertools import pairwise, product
+from itertools import product
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
@@ -66,7 +66,7 @@ def align_and_fill(
     )
 
 
-@minimum_version("pandas 2")
+@minimum_version("pandas 2; python 3.10")
 def smooth(c: Computer, key: "genno.Key", *, dim: str = "ya") -> "genno.Key":
     """Implement ‘smoothing’ for `key` along the dimension `dim`.
 
@@ -76,6 +76,8 @@ def smooth(c: Computer, key: "genno.Key", *, dim: str = "ya") -> "genno.Key":
     2. Remove those values.
     3. Fill by linear interpolation.
     """
+    from itertools import pairwise
+
     assert key.tag != "2"
     ks = KeySeq(key.remove_tag(key.tag or ""))
 

message_ix_models/model/transport/operator.py

@@ -4,7 +4,7 @@
 import re
 from collections.abc import Mapping, Sequence
 from functools import partial, reduce
-from itertools import pairwise, product
+from itertools import product
 from operator import gt, le, lt
 from typing import TYPE_CHECKING, Any, Hashable, Optional, Union, cast
 
@@ -26,6 +26,7 @@
 from message_ix_models.util import (
     MappingAdapter,
     datetime_now_with_tz,
+    minimum_version,
     nodes_ex_world,
     show_versions,
 )
@@ -953,6 +954,7 @@ def relabel2(qty: "AnyQuantity", new_dims: dict):
     return result
 
 
+@minimum_version("python 3.10")
 def uniform_in_dim(value: "AnyQuantity", dim: str = "y") -> "AnyQuantity":
     """Construct a uniform distribution from `value` along its :math:`y`-dimension.
 
@@ -967,6 +969,8 @@ def uniform_in_dim(value: "AnyQuantity", dim: str = "y") -> "AnyQuantity":
         and including :math:`d_{max}`. Values are the piecewise integral of the uniform
         distribution in the interval ending at the respective coordinate.
     """
+    from itertools import pairwise
+
     d_max = value.coords[dim].item()  # Upper end of the distribution
     width = 2 * value.item()  # Width of a uniform distribution
     height = 1.0 / width  # Height of the distribution

message_ix_models/testing/__init__.py

@@ -1,7 +1,11 @@
 import logging
 import os
 import shutil
-from base64 import b32hexencode
+
+try:
+    from base64 import b32hexencode as b32encode
+except ImportError:
+    from base64 import b32encode
 from collections.abc import Generator
 from copy import deepcopy
 from pathlib import Path
@@ -224,7 +228,7 @@ def bare_res(request, context: Context, solved: bool = False) -> message_ix.Scen
         new_name = request.node.name
     except AttributeError:
         # Generate a new scenario name with a random part
-        new_name = f"baseline {b32hexencode(randbytes(3)).decode().rstrip('=').lower()}"
+        new_name = f"baseline {b32encode(randbytes(3)).decode().rstrip('=').lower()}"
 
     log.info(f"Clone to '{model_name}/{new_name}'")
     return base.clone(scenario=new_name, keep_solution=solved)
@@ -425,7 +429,7 @@ def loaded_snapshot(
         new_name = request.node.name
     except AttributeError:
         # Generate a new scenario name with a random part
-        new_name = f"baseline {b32hexencode(randbytes(3)).decode().rstrip('=').lower()}"
+        new_name = f"baseline {b32encode(randbytes(3)).decode().rstrip('=').lower()}"
 
     log.info(f"Clone to '{model_name}/{new_name}'")
     yield base.clone(scenario=new_name, keep_solution=solved)

message_ix_models/tests/model/transport/test_config.py

@@ -75,18 +75,14 @@ def test_navigate_scenario1(self, c, input, expected):
 
     def test_scenario_conflict(self):
         # Giving both raises an exception
-        with pytest.raises(
-            ValueError,
-            match=r"SCENARIO.A___ and T35_POLICY.ACT\|TEC are not compatible",
-        ):
+        at = "(ACT|TEC)"  # Order differs in Python 3.9
+        expr = rf"SCENARIO.A___ and T35_POLICY.{at}\|{at} are not compatible"
+        with pytest.raises(ValueError, match=expr):
             c = Config(futures_scenario="A---", navigate_scenario="act+tec")
 
         # Also a conflict
         c = Config(navigate_scenario="act+tec")
-        with pytest.raises(
-            ValueError,
-            match=r"SCENARIO.A___ and T35_POLICY.ACT\|TEC are not compatible",
-        ):
+        with pytest.raises(ValueError, match=expr):
             c.set_futures_scenario("A---")
 
 

message_ix_models/tests/model/transport/test_operator.py

@@ -17,6 +17,7 @@
     factor_ssp,
     sales_fraction_annual,
     transport_check,
+    uniform_in_dim,
 )
 from message_ix_models.model.transport.structure import get_technology_groups
 from message_ix_models.project.navigate import T35_POLICY
@@ -139,6 +140,7 @@ def test_factor_ssp(test_context, ssp: SSP_2024) -> None:
 @pytest.mark.skipif(
     parse(version("genno")) < parse("1.25.0"), reason="Requires genno >= 1.25.0"
 )
+@uniform_in_dim.minimum_version
 def test_sales_fraction_annual():
     q = genno.Quantity(
         [[12.4, 6.1]], coords={"y": [2020], "n": list("AB")}, units="year"

message_ix_models/tests/report/test_compat.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 
 from genno import Computer
 from ixmp.testing import assert_logs
@@ -103,5 +104,11 @@ def test_prepare_techs_filter_error(caplog, monkeypatch):
     """:func:`.prepare_techs` logs warnings for invalid filters."""
     monkeypatch.setitem(TECH_FILTERS, "foo", "not a filter")
 
-    with assert_logs(caplog, "SyntaxError('invalid syntax", at_level=logging.WARNING):
+    message = "SyntaxError('" + (
+        "invalid syntax"
+        if sys.version_info >= (3, 10)
+        else "unexpected EOF while parsing"
+    )
+
+    with assert_logs(caplog, message, at_level=logging.WARNING):
         prepare_techs(Computer(), get_codes("technology"))

message_ix_models/types.py

@@ -37,4 +37,5 @@ class MaintainableArtefactArgs(TypedDict):
     is_external_reference: Optional[bool]
     is_final: Optional[bool]
     maintainer: Any
-    version: Optional[Union[str, sdmx.model.common.Version]]
+    # NB Only present from sdmx1 2.16; minimum for message-ix-models is sdmx1 2.13.1
+    version: Optional[Union[str, "sdmx.model.common.Version"]]

message_ix_models/util/__init__.py

@@ -573,16 +573,21 @@ def minimum_version(
     Parameters
     ----------
     expr :
-        Like "example 1.2.3.post0". The condition for the decorated function is that
-        the installed version must be equal to or greater than this version.
+        Like "pkgA 1.2.3.post0; pkgB 2025.2". The condition for the decorated function
+        is that the installed version must be equal to or greater than this version.
     """
+    from platform import python_version
 
     from packaging.version import parse
 
-    package, v_min = expr.split(" ")
-    v_package = version(package)
-    condition = parse(v_package) < parse(v_min)
-    message = f" with {package} {v_package} < {v_min}"
+    # Handle `expr`, updating `condition` and `message`
+    condition, message = False, " with "
+    for spec in expr.split(";"):
+        package, v_min = spec.strip().split(" ")
+        v_package = python_version() if package == "python" else version(package)
+        if parse(v_package) < parse(v_min):
+            condition = True
+            message += f"{package} {v_package} < {v_min}"
 
     # Create the decorator
     def decorator(func):

message_ix_models/util/_dataclasses.py

@@ -13,21 +13,28 @@
 import types
 from dataclasses import fields
 
+try:
+    from types import EllipsisType, NoneType, NotImplementedType
+except ImportError:  # Python 3.9
+    EllipsisType = type(...)  # type: ignore [misc]
+    NoneType = type(None)  # type: ignore [misc]
+    NotImplementedType = type(NotImplemented)  # type: ignore [misc]
+
 __all__ = [
     "asdict",
 ]
 
 _ATOMIC_TYPES = frozenset(
     {
-        types.NoneType,
+        NoneType,
         bool,
         int,
         float,
         str,
         complex,
         bytes,
-        types.EllipsisType,
-        types.NotImplementedType,
+        EllipsisType,
+        NotImplementedType,
         types.CodeType,
         types.BuiltinFunctionType,
         types.FunctionType,

message_ix_models/util/cache.py

@@ -32,6 +32,7 @@
 if TYPE_CHECKING:
     from pathlib import Path
 
+
 log = logging.getLogger(__name__)
 
 
@@ -46,11 +47,20 @@
 # Show genno how to hash function arguments seen in message_ix_models
 
 
-@genno.caching.Encoder.register
-def _quantity(o: AnyQuantity):
+def _quantity(o: "AnyQuantity"):
     return tuple(o.to_series().to_dict())
 
 
+try:
+    genno.caching.Encoder.register(AnyQuantity)(_quantity)
+except TypeError:  # Python 3.10 or earlier
+    from genno.core.attrseries import AttrSeries
+    from genno.core.sparsedataarray import SparseDataArray
+
+    genno.caching.Encoder.register(AttrSeries)(_quantity)
+    genno.caching.Encoder.register(SparseDataArray)(_quantity)
+
+
 # Upstream
 @genno.caching.Encoder.register
 def _enum(o: Enum):