Merge pull request #212 from iiasa/transport/2024-W31

Transport improvements for 2024-W31

Author: khaeru

.github/workflows/transport.yaml

@@ -1,6 +1,8 @@
 name: MESSAGEix-Transport
 
 env:
+  target: transport.yaml --repo=iiasa/message_data
+
   # Starting point of the workflow.
   # Use this value to build from a certain scenario:
   # base: --url="ixmp://ixmp-dev/MESSAGEix-GLOBIOM 1.1-R12/baseline_DEFAULT#21"
@@ -12,11 +14,11 @@ env:
   from-step: ""
 
 on:
-  # Uncomment these lines for debugging, but leave them commented on 'main'/'dev'
+  # Uncomment these lines for debugging, but leave them commented on 'main'
   # pull_request:
-  #   branches: [ main, dev ]
+  #   branches: [ main ]
   # push:
-  #   branches: [ main, dev ]
+  #   branches: [ main ]
   schedule:
   - cron: "0 1 * * *"  # 01:00 UTC = 02:00 CET = 03:00 CEST
   workflow_dispatch: {}
@@ -27,186 +29,24 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  report:
-    name: MESSAGEix-Transport ${{ matrix.SSP }}${{ matrix.policy }}
-    # Distribute jobs across any available self-hosted runner(s)
-    # runs-on: self-hosted
-    # Use one or a few specific runner(s)
-    runs-on: [ self-hosted, iiasa-self-hosted-l2 ]
-
-    strategy:
-      matrix:
-        # These correspond to short labels used for workflow step names
-        SSP:
-        - SSP1
-        - SSP2
-        - SSP3
-        - SSP4
-        - SSP5
-        policy:
-        - " policy"
-        - ""
-
-    steps:
-    - name: Clean workspace
-      run: |
-        rm -rf ${{ github.workspace }}/report/*
-
-    - uses: webfactory/[email protected]
-      with:
-        ssh-private-key: |
-          ${{ secrets.MESSAGE_BUILDINGS_PRIVATE_KEY }}
-          ${{ secrets.MESSAGE_TRADE_PRIVATE_KEY }}
-          ${{ secrets.NAVIGATE_WORKFLOW_PRIVATE_KEY }}
-
-    - uses: actions/checkout@v4
-      with:
-        path: message_data
-        lfs: true
-        # On GitHub Actions
-        # submodules: true
-        # On self-hosted runners
-        submodules: false
-
-    # NB Needed for self-hosted runner, not for GitHub
-    - name: Check out Git LFS files
-      working-directory: message_data
-      run: git lfs checkout
-
-    # Other repositories required for testing
-    - name: Check out message-static-data
-      uses: actions/checkout@v4
-      with:
-        repository: iiasa/message-static-data
-        token: ${{ secrets.MESSAGE_STATIC_DATA_PAT }}
-        path: message-static-data
-        lfs: true
-        # Only check out the following files to limit bandwidth usage.
-        # Respectively
-        # 1. IEA Extended World Energy Balances, OECD source, 2022 edition
-        sparse-checkout: |
-          iea/372f7e29-en.zip
-        sparse-checkout-cone-mode: false
-
-    - name: Connect static data to message_data
-      run: cp -rsv $(realpath message-static-data)/* message_data/data/
-
-    - uses: actions/checkout@v4
-      with:
-        repository: iiasa/MESSAGE_Buildings
-        path: buildings
-        ssh-key: ${{ secrets.MESSAGE_BUILDINGS_PRIVATE_KEY }}
-
-    # NB Needed for self-hosted runner, not for GitHub
-    - uses: actions/setup-java@v4
-      with:
-        distribution: temurin
-        java-version: "17"
-
-    - uses: actions/setup-python@v5
-      with:
-        # Latest version testable on GitHub Actions
-        python-version: "3.12"
-        cache: pip
-        cache-dependency-path: "**/pyproject.toml"
-
-    - uses: iiasa/actions/setup-gams@main
-      with:
-        version: 35.2.0
-        license: ${{ secrets.GAMS_LICENSE }}
-
-    # NB the IIASA-hosted runner does not have permissions to do "sudo apt-get",
-    #    so the following doesn't work. Rather, the necessary packages
-    #    (graphviz libgraphviz-dev pkg-config) are manually installed on the
-    #    host system
-    # - uses: ts-graphviz/setup-graphviz@v2
-
-    # By default, the next step installs dependencies from PyPI.
-    # To run against unreleased code (on `main`, or other branches for open
-    # PRs), uncomment or modify the lines below as appropriate.
-    # - genno and message-ix-models are released frequently and *should* be
-    #   installed from PyPI, with only temporary exceptions.
-    # - ixmp and message-ix are released less frequently and *may* be installed
-    #   from `main`
-    - name: Force upgrade to unreleased dependencies
-      run: |
-        # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
-        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@main"
-        pip install --upgrade "message_ix @ git+https://github.com/iiasa/message_ix.git@main"
-        # Temporary, pending a release that includes iiasa/message-ix-models#161, #166
-        pip install --upgrade "message-ix-models[iea-web] @ git+https://github.com/iiasa/message-ix-models.git@main"
-
-        # Upgrade matplotlib
-        # pyam pins matplotlib < 3.7.1, which conflicts with plotnine per
-        # https://github.com/has2k1/plotnine/pull/751. Override.
-        pip install --upgrade matplotlib
-
-    - name: Install package; upgrade dependencies
-      working-directory: message_data
-      run: pip install --upgrade --upgrade-strategy=eager --editable .[dl,scgen,tests,transport]
-
-    - name: Configure
-      run: |
-        from pathlib import Path
-
-        # Ensure keys are defined
-        import message_data.model.buildings
-        from ixmp import config
-
-        config.add_platform(
-            "ixmp-dev",
-            "jdbc",
-            "oracle",
-            "x8oda.iiasa.ac.at:1521/PIXMP2.iiasa.ac.at",
-            "${{ secrets.IXMP_DEV_USER }}",
-            "${{ secrets.IXMP_DEV_PASS }}",
-            jvmargs="-Xmx16G",
-        )
-
-        p = Path("${{ github.workspace }}")
-        config.set("message local data", p / "report")
-        config.set("message solve options", {"iis": 1, "tilim": 60 * 30})
-        config.set("message buildings dir", p / "buildings")
-        config.save()
-
-        # Equivalent to "(ixmp|message-ix|mix-models) config show"
-        print(config.path.read_text())
-      shell: python
-
-    - name: Fetch data
-      run: |
-        mix-models fetch "SSP-Update-3.0"
-        mix-models fetch "SSP-Update-3.0.1"
-
-    - name: Run MESSAGEix-Transport workflow "${{ env.from-step }}" → "${{ matrix.SSP }}${{ matrix.policy }} reported"
-      run: |
-        mix-models \
-          ${{ env.base }} \
-          transport run \
-          --base=auto --nodes=R12 --model-extra="ci nightly" \
-          --from="${{ env.from-step }}" \
-          "${{ matrix.SSP }}${{ matrix.policy }} reported" \
-          --go
-      continue-on-error: true
-
-    - name: Upload results as a build artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: MESSAGEix-Transport ${{ matrix.SSP }}${{ matrix.policy }}
-        path: |
-          ${{ github.workspace }}/report/
-          !${{ github.workspace }}/report/cache/
-        if-no-files-found: error
-      continue-on-error: true
-
-  collate:
-    name: Collate outputs
-    needs: report
-    runs-on: self-hosted
+  dispatch:
+    runs-on: ubuntu-latest
 
     steps:
-    - name: Merge artifacts
-      uses: actions/upload-artifact/merge@v4
-      with:
-        name: MESSAGEix-Transport
-        delete-merged: true
+      - name: Invoke "transport.yaml" workflow in message_data
+        env:
+          GH_TOKEN: ${{ secrets.MESSAGE_DATA_DISPATCH_TOKEN }}
+        run: |
+          gh workflow run \
+            ${{ env.target }} --ref fix/transport-workflow \
+            --field ref=${{ github.ref }} \
+            --field sha=${{ github.sha }} \
+            --field base=${{ env.base }} \
+            --field from-step=${{ env.from-step }}
+
+          sleep 5
+
+          gh run list \
+            --workflow=${{ env.target }} \
+            --json url,status \
+            --jq 'map(select(.status != "completed"))[0].url' >> $GITHUB_STEP_SUMMARY

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
     language: python
     entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@; python -m pip list"
     additional_dependencies:
-    - mypy >= 1.9.0
+    - mypy >= 1.11.0
     - plotnine
     - pytest
     - sdmx1
@@ -20,7 +20,7 @@ repos:
     - "message-ix @ git+https://github.com/iiasa/message_ix.git@main"
     args: ["."]
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.3.2
+  rev: v0.5.5
   hooks:
   - id: ruff
   - id: ruff-format

MANIFEST.in

@@ -1,10 +1,15 @@
+# Repository files for development, not distribution
+exclude .*
+prune .github
+
 # Data for testing
 prune message_ix_models/data/test/advance
 prune message_ix_models/data/test/gea
 prune message_ix_models/data/test/iea
 prune message_ix_models/data/test/shape
 prune message_ix_models/data/test/snapshot-*
 prune message_ix_models/data/test/ssp
+
 # Larger package data
 # - Not distributed on PyPI.
 # - Should be fetched with Pooch from GitHub.

doc/transport/input.rst

@@ -7,8 +7,7 @@ This page describes the structure and format of inputs required for building MES
    :local:
    :backlinks: none
 
-Both input data and configuration are stored in files under :file:`/data/transport/` in the :mod:`message_data` repository.
-(When migrated to :mod:`message_ix_models`, these files will live in :file:`message_ix_models/data/transport`.)
+Both input data and configuration are stored in files under :file:`message_ix_models/data/transport/`.
 
 In most cases, these files are read from a subdirectory like :file:`/data/transport/{nodes}/`, where `nodes` denotes the :mod:`message_ix_models` :doc:`node code list </pkg-data/node>`—for instance, "R12"—for which MESSAGEix-Transport will be built.
 This value is retrieved from the :attr:`Context.regions <.model.Config.regions>` setting.
@@ -66,7 +65,7 @@ Through :func:`.transport.build.main` (ultimately, :func:`.transport.build.add_e
    The file has column headers "node", "technology", and "value".
 
 Not all files are currently or always used in model-building computations.
-Some submodules of :mod:`~message_data.model.transport` use additional data files via other mechanisms.
+Some submodules of :mod:`~message_ix_models.model.transport` use additional data files via other mechanisms.
 Most of the files have a header comment including a precise description of the quantity, source of the data, and units of measurement; in some cases extended information is below (where a header comment would be too long).
 
 :file:`ldv-activity.csv` → ``ldv activity:n:exo``
@@ -224,7 +223,7 @@ Other files
 Other data sources
 ==================
 
-:mod:`~message_data.model.transport` makes use of the :mod:`message_ix_models.tools.exo_data` mechanism to retrieve data from common (not transport-specific) sources.
+:mod:`~message_ix_models.model.transport` makes use of the :mod:`message_ix_models.tools.exo_data` mechanism to retrieve data from common (not transport-specific) sources.
 :class:`.DataSourceConfig`, :attr:`.transport.Config.ssp`, and other settings determine which sources and quantities are used.
 
 These include:

message_ix_models/model/transport/CHN_IND.py

@@ -67,7 +67,7 @@ def _convert(group_df):
     return group_df.assign(Value=qty.magnitude, Units=qty.units)
 
 
-def split_variable(s):
+def split_variable(s) -> pd.DataFrame:
     """Split strings in :class:`pandas.Series` *s* into Variable and Mode.
 
     Parameters
@@ -97,7 +97,7 @@ def split_variable(s):
     return df
 
 
-def get_ind_item_data():
+def get_ind_item_data() -> pd.DataFrame:
     """Retrieve activity data for rail and road transport for India from iTEM.
 
     Data is obtained from iTEM database's file ``T000.csv`` and filtered for the
@@ -134,7 +134,7 @@ def get_ind_item_data():
     return all_data
 
 
-def get_chn_ind_pop():
+def get_chn_ind_pop() -> pd.DataFrame:
     """Retrieve population data for China and India.
 
     The dataset is a ``.csv`` file in */data* and was retrieved from `OECD
@@ -157,7 +157,7 @@ def get_chn_ind_pop():
     return pop
 
 
-def get_chn_ind_data(private_vehicles=False):
+def get_chn_ind_data(private_vehicles=False) -> pd.DataFrame:
     """Read transport activity and vehicle stock data for China and India.
 
     The data is read from ``data/transport`` folder (data for China from NBSC) and

message_ix_models/model/transport/roadmap.py

@@ -83,7 +83,9 @@
 )
 
 
-def get_roadmap_data(context, region=("Africa", "R11_AFR"), years=None, plot=False):
+def get_roadmap_data(
+    context, region=("Africa", "R11_AFR"), years=None, plot=False
+) -> pd.DataFrame:
     """Read transport activity data for Africa.
 
     The data is read from ``RoadmapResults_2017.xlsx``, which is already aggregated

message_ix_models/tests/model/transport/test_emission.py

@@ -56,7 +56,7 @@ def test_ef_for_input(test_context):
     ra = result["relation_activity"]
     assert not ra.isna().any(axis=None), ra
 
-    assert int == ra.dtypes["year_act"]
+    assert ra.dtypes["year_act"] == int  # noqa: E721
 
     # print(ra.to_string())
 

pyproject.toml

@@ -161,7 +161,7 @@ select = ["C9", "E", "F", "I", "W"]
 # - .util.sdmx.as_codes(): 13 > 11
 mccabe.max-complexity = 11
 
-[tool.setuptools.packages.find]
-include = ["message_ix_models*"]
+[tool.setuptools.packages]
+find = {}
 
 [tool.setuptools_scm]