Merge pull request #288 from iiasa/fix/ci-checkout-ref

Check out correct ref in "pytest" CI workflow

Author: khaeru

.github/workflows/pytest.yaml

@@ -1,7 +1,7 @@
 name: Test
 
 on:
-  # Uncomment this entry only to debug the workflow
+  # To debug the workflow, uncomment this entry AND comment pull_request_target
   # pull_request:
   #   branches: [ main ]
   pull_request_target:
@@ -26,7 +26,7 @@ env:
 
 jobs:
   check:
-    name: check permissions
+    name: Check permissions, identify ref to test
     runs-on: ubuntu-latest
     steps:
     - if: >
@@ -39,7 +39,15 @@ jobs:
         echo "Pytest workflow will not run for branch in fork without label \`${{ env.label }}\`." >>$GITHUB_STEP_SUMMARY
         exit 1
 
+    - name: Identify ref to check out
+      id: ref
+      run: echo "ref=${{ github.event_name != 'pull_request_target' && github.ref || github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT
+
+    outputs:
+      ref: ${{ steps.ref.outputs.ref }}
+
   warm-lfs-cache:
+    name: Warm Git LFS cache
     needs: check
 
     strategy:
@@ -48,6 +56,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: francisbilham11/action-cached-lfs-checkout@v3
+      with:
+        ref: ${{ needs.check.outputs.ref }}
 
   pytest:
     needs: warm-lfs-cache
@@ -64,8 +74,6 @@ jobs:
         # - Versions of ixmp and message_ix to test.
         # - Latest supported Python version for those or other dependencies.
         # Minimum version given in pyproject.toml + earlier version of Python
-        - { upstream: v3.4.0, python: "3.11" }  # 2022-01-27
-        - { upstream: v3.5.0, python: "3.11" }  # 2022-05-06
         - { upstream: v3.6.0, python: "3.11" }  # 2022-08-18
         - { upstream: v3.7.0, python: "3.11" }  # 2023-05-17
         - { upstream: v3.8.0, python: "3.12" }  # 2024-01-12
@@ -77,14 +85,12 @@ jobs:
         exclude:
         # Specific version combinations that are invalid / not to be used
         # These versions of ixmp are not able locate the arm64 GAMS API binaries
-        - { os: macos-latest, version: {upstream: v3.4.0 }}
-        - { os: macos-latest, version: {upstream: v3.5.0 }}
-        - { os: macos-latest, version: {upstream: v3.6.0 }}
-        - { os: macos-latest, version: {upstream: v3.7.0 }}
-        - { os: macos-latest, version: {upstream: v3.8.0 }}
-        - { os: macos-latest, version: {upstream: v3.9.0 }}
+        - { os: macos-latest, version: { upstream: v3.6.0 }}
+        - { os: macos-latest, version: { upstream: v3.7.0 }}
+        - { os: macos-latest, version: { upstream: v3.8.0 }}
+        - { os: macos-latest, version: { upstream: v3.9.0 }}
         # Redundant with macos-latest
-        - { os: macos-13, version: {upstream: main }}
+        - { os: macos-13, version: { upstream: main }}
 
       fail-fast: false
 
@@ -100,6 +106,8 @@ jobs:
 
     - name: Check out message-ix-models
       uses: francisbilham11/action-cached-lfs-checkout@v3
+      with:
+        ref: ${{ needs.check.outputs.ref }}
 
     - name: Set up uv, Python
       uses: astral-sh/setup-uv@v5
@@ -171,11 +179,12 @@ jobs:
 
   pre-commit:
     name: Code quality
-
+    needs: check
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v4
+      with:
+        ref: ${{ needs.check.output.ref }}
     - uses: astral-sh/setup-uv@v5
       with:
         cache-dependency-glob: "**/pyproject.toml"

README.rst

@@ -10,7 +10,7 @@ See the `online documentation <https://docs.messageix.org/projects/models/>`_ fo
 License
 =======
 
-Copyright © 2020–2024 IIASA Energy, Climate, and Environment (ECE) program
+Copyright © 2020–2025 IIASA Energy, Climate, and Environment (ECE) program
 
 Licensed under the Apache License, version 2.0.
 

doc/conf.py

@@ -13,7 +13,7 @@
 # -- Project information ---------------------------------------------------------------
 
 project = "message-ix-models"
-copyright = "2020–2024, IIASA Energy, Climate, and Environment (ECE) Program"
+copyright = "2020–%Y, IIASA Energy, Climate, and Environment (ECE) Program"
 author = "IIASA Energy, Climate, and Environment (ECE) Program"
 
 

doc/whatsnew.rst

@@ -4,6 +4,8 @@ What's new
 Next release
 ============
 
+- Support for :mod:`ixmp` and :mod:`message_ix` versions 3.4.x and 3.5.x is dropped in accordance with the :ref:`policy-upstream-versions` (:pull:`288`).
+  The minimum version of both packages is 3.6.0.
 - Update :class:`.IEA_EWEB` to support :py:`transform="B"` / :func:`.transform_B` (:issue:`230`, :pull:`259`).
 - Update :doc:`/transport/index` (:pull:`259`).
 

message_ix_models/model/transport/config.py

@@ -341,12 +341,15 @@ def from_context(
         # Handle arguments
         options = options or dict()
 
+        log.debug(f".transport.Config.from_context: {context.model.regions = }")
         try:
             # Identify the node codelist used in `scenario`
             regions = identify_nodes(scenario) if scenario else context.model.regions
         except (AttributeError, ValueError):
             pass
         else:
+            if scenario:
+                log.debug(f".transport.Config.from_context: {scenario.set('node') = }")
             if context.model.regions != regions:
                 log.info(
                     f"Override Context.model.regions={context.model.regions!r} with "

message_ix_models/model/transport/ldv.py

@@ -299,7 +299,7 @@ def prepare_tech_econ(
 
     # Create base quantity for "output" parameter
     nty = tuple("nty")
-    c.add(k.output[0] * nty, wildcard(1.0, "", nty))
+    c.add(k.output[0] * nty, wildcard(1.0, "Gv km", nty))
     for i, coords in enumerate(["n::ex world", "t::LDV", "y::model"]):
         c.add(
             k.output[i + 1] * nty,

message_ix_models/model/transport/testing.py

@@ -2,7 +2,7 @@
 
 import logging
 import platform
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Hashable, Mapping
 from contextlib import nullcontext
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Union
@@ -14,7 +14,7 @@
 from message_ix_models import Context, ScenarioInfo
 from message_ix_models.report.sim import add_simulated_solution
 from message_ix_models.testing import GHA, bare_res
-from message_ix_models.util import silence_log
+from message_ix_models.util import identify_nodes, silence_log
 from message_ix_models.util.graphviz import HAS_GRAPHVIZ
 
 from . import Config, build
@@ -28,12 +28,12 @@
 
 # Common marks for transport code. Do not reuse keys that are less than the highest key
 # appearing in the dict.
-MARK: dict[int, pytest.MarkDecorator] = {
+MARK: dict[Hashable, pytest.MarkDecorator] = {
     0: pytest.mark.xfail(
-        reason="Missing R14 input data/assumptions", raises=FileNotFoundError
+        reason="Missing R14 input data/config", raises=FileNotFoundError
     ),
     1: pytest.mark.skip(
-        reason="Currently only possible with regions=R12 input data/assumptions",
+        reason="Currently only possible with regions=R12 input data/config",
     ),
     3: pytest.mark.xfail(raises=ValueError, reason="Missing ISR/mer-to-ppp.csv"),
     4: pytest.mark.xfail(reason="Currently unsupported"),
@@ -42,10 +42,14 @@
         condition=GHA and platform.system() == "Darwin" and not HAS_GRAPHVIZ,
         reason="Graphviz missing on macos-13 GitHub Actions runners",
     ),
-    8: pytest.mark.xfail(
+    "gh-281": pytest.mark.xfail(
         raises=ModelError,
         reason="Temporary, for https://github.com/iiasa/message-ix-models/pull/281",
     ),
+    9: pytest.mark.xfail(reason="Missing R14 input data/config"),
+    "gh-288": pytest.mark.xfail(
+        reason="Temporary, for https://github.com/iiasa/message-ix-models/pull/288",
+    ),
 }
 
 make_mark: dict[int, Callable[..., pytest.MarkDecorator]] = {
@@ -115,7 +119,7 @@ def built_transport(
     model_name = res.model.replace("-GLOBIOM", "-Transport")
 
     try:
-        scenario = Scenario(context.get_platform(), model_name, "baseline")
+        scenario = Scenario(res.platform, model_name, "baseline")
     except ValueError:
         log.info(f"Create '{model_name}/baseline' for testing")
 
@@ -138,7 +142,20 @@ def built_transport(
         scenario.solve(solve_options=dict(lpmethod=4))
 
     log.info(f"Clone to '{model_name}/{request.node.name}'")
-    return scenario.clone(scenario=request.node.name, keep_solution=solved)
+    result = scenario.clone(scenario=request.node.name, keep_solution=solved)
+
+    if (
+        GHA
+        and platform.system() == "Darwin"
+        and identify_nodes(result) != context.model.regions
+    ):
+        pytest.xfail(
+            reason="Known issue on GitHub Actions macOS runners: result has nodes "
+            f"{identify_nodes(result) = !r} != {identify_nodes(res) = !r} == "
+            f"{context.model.regions = !r}"
+        )
+
+    return result
 
 
 def simulated_solution(request, context) -> Reporter:
@@ -190,4 +207,6 @@ def simulated_solution(request, context) -> Reporter:
     with silence_log("genno", logging.CRITICAL):
         message_ix_models.report.prepare_reporter(context, reporter=rep)
 
+    log.debug(f"simulated_solution: {context.regions = }")
+
     return rep

message_ix_models/project/ssp/cli.py

@@ -69,4 +69,4 @@ def transport_cmd(context: "Context", method, path_in: Path, path_out: Optional[
 
     if path_out_user != path_out:
         print(f"Convert CSV output to {path_out_user}")
-        pd.read_csv(path_out).to_excel(path_out_user)
+        pd.read_csv(path_out).to_excel(path_out_user, index=False)

message_ix_models/report/__init__.py

@@ -281,6 +281,8 @@ def prepare_reporter(
     """
     log.info("Prepare reporter")
 
+    log.debug(f".report.prepare_reporter: {context.regions = }")
+
     if reporter:
         # Existing `Reporter` provided
         rep = reporter

message_ix_models/testing/__init__.py

@@ -202,6 +202,7 @@ def bare_res(request, context: Context, solved: bool = False) -> message_ix.Scen
     from message_ix_models.model import bare
 
     # Model name: standard "MESSAGEix-GLOBIOM R12 YB" plus a suffix
+    log.info(f"bare_res: {context.model.regions = }")
     model_name = bare.name(context, unique=True)
 
     mp = context.get_platform()
@@ -213,6 +214,8 @@ def bare_res(request, context: Context, solved: bool = False) -> message_ix.Scen
         context.scenario_info.update(model=model_name, scenario="baseline")
         base = bare.create_res(context)
 
+    log.info(f"bare_res: {base.set('node') = }")
+
     if solved and not base.has_solution():
         log.info("Solve")
         base.solve(solve_options=dict(lpmethod=4), quiet=True)

message_ix_models/tests/model/transport/test_build.py

@@ -91,7 +91,7 @@ def test_make_spec(regions_arg, regions_exp, years):
         param("R11", "B", False, "IKARUS", False, marks=[mark.slow, MARK[1]]),
         param("R11", "B", False, "IKARUS", True, marks=[mark.slow, MARK[1]]),
         # R12, B
-        param("R12", "B", False, "IKARUS", True, marks=MARK[8]),
+        param("R12", "B", False, "IKARUS", True, marks=MARK["gh-281"]),
         # R14, A
         param(
             "R14",
@@ -276,33 +276,33 @@ def test_build_existing(tmp_path, test_context, url, solve=False):
 @pytest.mark.parametrize(
     "build_kw",
     (
-        dict(regions="R11", years="A", options=dict()),
-        dict(regions="R11", years="B", options=dict()),
+        dict(regions="R11", years="A"),
+        dict(regions="R11", years="B"),
         dict(regions="R11", years="B", options=dict(futures_scenario="A---")),
         dict(regions="R11", years="B", options=dict(futures_scenario="debug")),
-        dict(regions="R12", years="B", options=dict()),
+        dict(regions="R12", years="B"),
         dict(regions="R12", years="B", options=dict(navigate_scenario="act+ele+tec")),
-        dict(regions="R14", years="B", options=dict()),
-        param(dict(regions="ISR", years="A", options=dict()), marks=MARK[3]),
+        param(dict(regions="R14", years="B"), marks=MARK[9]),
+        param(dict(regions="ISR", years="A"), marks=MARK[3]),
     ),
 )
 def test_debug(
     test_context,
     tmp_path,
     build_kw,
     N_node,
-    verbosity: Literal[0, 1, 2, 3] = 2,  # NB Increase this to show more verbose output
+    verbosity: Literal[0, 1, 2, 3] = 0,  # NB Increase this to show more verbose output
 ):
     """Debug particular calculations in the transport build process."""
     # Get a Computer prepared to build the model with the given options
     c, info = configure_build(test_context, tmp_path=tmp_path, **build_kw)
 
     # Construct a list of common checks
     verbose: dict[int, list["Check"]] = {
-        0: [],
-        1: [Log(7)],
-        2: [Log(None)],
-        3: [Dump(tmp_path)],
+        0: [],  # Don't log anything.
+        1: [Log()],  # Log 7 lines at the start/end of each quantity.
+        2: [Log(None)],  # Log *all* data. This is produces GHA logs >1 GiB.
+        3: [Dump(tmp_path)],  # Dump all data to a file.
     }
     common = [Size({"n": N_node}), NoneMissing()] + verbose[verbosity]
 

message_ix_models/tests/model/transport/test_data.py

@@ -2,7 +2,7 @@
 import pytest
 from iam_units import registry
 
-from message_ix_models.model.transport import build, testing
+from message_ix_models.model.transport import build, non_ldv, testing
 from message_ix_models.model.transport.CHN_IND import get_chn_ind_data, get_chn_ind_pop
 from message_ix_models.model.transport.roadmap import get_roadmap_data
 from message_ix_models.model.transport.testing import MARK, assert_units, make_mark
@@ -73,7 +73,7 @@ def test_get_non_ldv_data(test_context, regions, years="B"):
     c, _ = testing.configure_build(ctx, regions=regions, years=years)
 
     # Code runs
-    data = c.get("transport nonldv::ixmp")
+    data = c.get(f"transport{non_ldv.Pi}")
 
     # Data are provided for the these parameters
     exp_pars = {
@@ -189,7 +189,13 @@ def test_get_chn_ind_pop():
 @build.get_computer.minimum_version
 @pytest.mark.parametrize("years", ["A", "B"])
 @pytest.mark.parametrize(
-    "regions", [pytest.param("ISR", marks=MARK[3]), "R11", "R12", "R14"]
+    "regions",
+    [
+        pytest.param("ISR", marks=MARK[3]),
+        "R11",
+        "R12",
+        pytest.param("R14", marks=MARK[9]),
+    ],
 )
 @pytest.mark.parametrize("options", [{}, dict(navigate_scenario=T35_POLICY.ELE)])
 def test_navigate_ele(test_context, regions, years, options):

message_ix_models/tests/model/transport/test_ikarus.py

@@ -17,7 +17,7 @@
     [
         ("R11", 11),
         ("R12", 12),
-        ("R14", 14),
+        pytest.param("R14", 14, marks=testing.MARK[9]),
         pytest.param("ISR", 1, marks=testing.MARK[3]),
     ],
 )