Expand docs for #21; add to whatsnew

Author: khaeru

doc/api.in

@@ -14,6 +14,7 @@
    oica
    org
    proto
+   report
    store
    testing
    tests

doc/conf.py

@@ -33,6 +33,15 @@
 nitpicky = True
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
+# A string of reStructuredText included at the beginning of every source file.
+rst_prolog = r"""
+.. role:: py(code)
+   :language: python
+
+.. role:: xml(code)
+   :language: xml
+"""
+
 
 def setup(app: "sphinx.application.Sphinx"):
     from sphinx.ext.autosummary.generate import generate_autosummary_docs
@@ -72,8 +81,11 @@ def setup(app: "sphinx.application.Sphinx"):
 
 intersphinx_mapping = {
     "click": ("https://click.palletsprojects.com/en/8.1.x/", None),
+    "docutils": ("https://sphinx-docutils.readthedocs.io/en/latest", None),
+    "jinja2": ("https://jinja.palletsprojects.com/en/3.1.x", None),
     "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
     "platformdirs": ("https://platformdirs.readthedocs.io/en/latest/", None),
+    "pluggy": ("https://pluggy.readthedocs.io/en/stable", None),
     "pooch": ("https://www.fatiando.org/pooch/latest/", None),
     "py": ("https://docs.python.org/3/", None),
     "pytest": ("https://docs.pytest.org/en/stable/", None),

doc/giz.rst

@@ -5,8 +5,10 @@ GIZ GmbH (`website <https://www.giz.de/en/>`_, lit. *Corporation for Internation
 It is not currently a direct provider of (meta)data through TDC, but its members initiated what is now the TDCI and support its activities, including development of this :mod:`.transport_data` package.
 This work mainly appears in the :mod:`.org` and :mod:`.proto` modules.
 
+.. _project-tuewas:
+
 TUEWAS
 ======
 
 - “Transport, Environment, Energy, and Water in Asia” is an “internal sector network” of GIZ.
-- Website: https://tuewas-asia.org/
+- Website: https://tuewas-asia.org

doc/index.rst

@@ -84,13 +84,15 @@ The following modules contain *generic* code and utilities usable with (meta)dat
 
    cli
    config
+   report
    store
    testing
    tests
    util
 
 - :mod:`~transport_data.cli`: :doc:`cli`
 - :mod:`.config`: :doc:`config`
+- :mod:`.report`: :doc:`report`
 - :mod:`.store`: :doc:`store`
 - :mod:`.testing`: :doc:`testing`
 - :mod:`.tests`: :doc:`tests`

doc/report.rst

@@ -0,0 +1,4 @@
+Reports
+*********
+
+.. include:: _api/transport_data.report.rst

doc/roadmap.rst

@@ -2,4 +2,7 @@ Roadmap
 *******
 
 This page gives a medium- and long-term overview of future development of :mod:`transport_data`, focused on the tools in this package, but with relevant details about the broader TDC and TDCI.
-See also `transport-data/projects/1 <https://github.com/orgs/transport-data/projects/1>`_ on GitHub.
+See also:
+
+- GitHub project `transport-data/projects/1 <https://github.com/orgs/transport-data/projects/1>`_ for general TDCI development.
+- GitHub project `transport-data/projects/3 <https://github.com/orgs/transport-data/projects/3>`_ for deployment of CKAN (2024).

doc/whatsnew.rst

@@ -4,6 +4,14 @@ What's new
 Next release
 ============
 
+- Add tools and data for the :ref:`project-tuewas` project (:pull:`21`).
+
+  - Add :mod:`.metadata.spreadsheet`,  :mod:`.metadata.report` submodules; expand :mod:`.metadata`.
+  - Add :program:`tdc org read`, :program:`tdc org summarize`, :program:`tdc org tuewas` CLI commands.
+
+- Add :class:`.report.Report`, a base class for generating ‘reports’ (documents derived from SDMX (meta)data) and supporting code in :mod:`.util.docutils`, :mod:`.util.jinja2` (:pull:`21`).
+- Adopt :mod:`pluggy <.util.pluggy>` for plug-in hooks and implementations (:pull:`21`); use the :func:`.hooks.get_agencies` hook across existing modules.
+- Add :func:`.tdc_cli`, :func:`.test_data_path` test fixtures (:pull:`21`).
 - Python 3.8 support is dropped (:pull:`21`), as it has reached end-of-life.
 - Add :mod:`.ipcc` (:doc:`ipcc`) module (:issue:`15`, :pull:`21`).
 - Add :doc:`standards` and :doc:`roadmap` documentation pages (:pull:`9`).

transport_data/org/__init__.py

@@ -15,6 +15,7 @@
 
 @hookimpl
 def get_agencies() -> "sdmx.model.v21.Agency":
+    """Return agencies and organizations including and subsidiary to TDCI itself."""
     # Agency
     a1 = m.Agency(
         id="TDCI",

transport_data/org/cli.py

@@ -1,5 +1,8 @@
 """CLI for :mod:`.org`.
 
+Use the :program:`--help` command-line option to see help for individual commands
+defined in this module.
+
 .. runblock:: console
 
    $ tdc org --help

transport_data/org/metadata/__init__.py

@@ -1,3 +1,5 @@
+"""Handle TDC-structured metadata."""
+
 import itertools
 import logging
 import re
@@ -174,6 +176,7 @@ def get_cs_common() -> "common.ConceptScheme":
 
 
 def get_msd() -> "v21.MetadataStructureDefinition":
+    """Generate and return the TDC metadata structure definition."""
     from transport_data import STORE
     from transport_data.org import get_agencyscheme
 

transport_data/org/metadata/report.py

@@ -1,3 +1,21 @@
+"""Generate reports about TDC-structured metadata.
+
+:class:`.Report` subclasses in this file **should** have names like::
+
+    { Type }{ ID }{ Format }
+
+…wherein:
+
+- ``{ Type }`` refers to the type of object(s) from the SDMX Information Model that
+  is/are represented in the report.
+  Usually the first argument to the :py:`__init__()` method is an instance of this type.
+- ``{ ID }`` is a number that distinguishes different ‘kinds’ of reports for the same
+  ``{ Type }``.
+  Report classes with the same ``{ Type }{ ID }`` **should** display roughly the same
+  information in the same order and layout, regardless of ``{ Format }``.
+- ``{ Format }`` is the output format or file type.
+"""
+
 from dataclasses import dataclass
 from functools import partial
 from typing import TYPE_CHECKING
@@ -11,9 +29,15 @@
 
 @dataclass
 class MetadataAttribute0Plain(Report):
-    """Summarize unique values appearing in `mds` for attribute `mda_id`."""
+    """Unique values appearing in `mds` for metadata attribute `mda_id`.
+
+    Each unique value is shown with the IDs of the data flows that contain the value for
+    `mda_id`.
+    """
 
+    #: Metadata set to report.
     mds: "v21.MetadataSet"
+    #: ID of a Metadata Attribute to report.
     mda_id: str
 
     def render(self) -> str:
@@ -34,10 +58,16 @@ def render(self) -> str:
 
 @dataclass
 class MetadataAttribute0RST(Report):
-    """Summarize unique values appearing in `mds` for attribute `mda_id`."""
+    """Unique values appearing in `mds` for the metadata attribute `mda_id`.
 
+    Same as :class:`.MetadataAttribute0Plain`, but in reStructuredText.
+    """
+
+    #: Jinja2 reStructuredText template.
     template_name = "metadata-attribute-0.rst"
+    #: Metadata set to report.
     mds: "v21.MetadataSet"
+    #: ID of a Metadata Attribute to report.
     mda_id: str
 
     def render(self) -> str:
@@ -52,6 +82,9 @@ def render(self) -> str:
 
 @dataclass
 class MetadataReport0HTML(Report):
+    """Same as :class:`.MetadataReport0Plain`, but in HTML."""
+
+    #: Metadata report to report.
     mdr: "v21.MetadataReport"
 
     def render(self) -> str:
@@ -60,6 +93,15 @@ def render(self) -> str:
 
 @dataclass
 class MetadataReport0Plain(Report):
+    """Contents of a single metadata report.
+
+    This includes:
+
+    1. The reported attribute values for all metadata attributes.
+    2. The data flow that is targeted by the report and its dimensions.
+    """
+
+    #: Metadata report to report.
     mdr: "v21.MetadataReport"
 
     def render(self) -> str:
@@ -100,9 +142,15 @@ def render(self) -> str:
 
 @dataclass
 class MetadataSet0ODT(Report):
-    """Print a summary of the contents of `mds`."""
+    """Summary of the unique reported attribute values in `mds`.
+
+    Similar to :class:`.MetadataSet0Plain`, but also including:
+
+    - The unique dimension concepts appearing in the data structure definitions.
+    """
 
     template_name = "metadata-set-0.rst"
+    #: Metadata set to report.
     mds: "v21.MetadataSet"
 
     def render(self) -> bytes:
@@ -124,8 +172,15 @@ def render(self) -> bytes:
 
 @dataclass
 class MetadataSet0Plain(Report):
-    """Print a summary of the contents of `mds`."""
+    """Summary of the unique reported attribute values in `mds`.
+
+    This includes:
 
+    - Unique values of the metadata attributes ``MEASURE``, ``DATA_PROVIDER``, and
+      ``UNIT_MEASURE``.
+    """
+
+    #: Metadata set to report.
     mds: "v21.MetadataSet"
 
     def render(self) -> str:
@@ -144,7 +199,7 @@ def render(self) -> str:
 
 @dataclass
 class MetadataSet1HTML(Report):
-    """Generate a summary report in HTML."""
+    """Metadata reports related to `ref_area`."""
 
     template_name = "metadata-set-1.html"
 
@@ -166,31 +221,12 @@ def render(self) -> str:
 
 
 @dataclass
-class MetadataSet2HTML(Report):
-    """Generate a summary report in HTML."""
-
-    template_name = "metadata-set-2.html"
-
-    #: Metadata set to summarize.
-    mds: "v21.MetadataSet"
-    #: Geography.
-    ref_area: list[str]
-
-    def render(self) -> str:
-        from transport_data.org.metadata import contains_data_for
-
-        data = {
-            mdr.attaches_to.key_values["DATAFLOW"].obj.id: {  # type: ignore [union-attr]
-                ra: contains_data_for(mdr, ra) for ra in self.ref_area
-            }
-            for mdr in self.mds.report
-        }
-
-        return self.render_jinja_template(ref_area=self.ref_area, data=data)
+class MetadataSet1ODT(Report):
+    """Metadata reports related to `ref_area`.
 
+    Same as :class:`.MetadataSet1HTML` but as OpenDocument Text.
+    """
 
-@dataclass
-class MetadataSet1ODT(Report):
     template_name = "metadata-set-1.rst"
 
     #: Metadata set to summarize.
@@ -216,3 +252,35 @@ def render(self) -> bytes:
 
         # Convert reStructuredText → OpenDocumentText
         return self.rst2odt(rst_source)
+
+
+@dataclass
+class MetadataSet2HTML(Report):
+    """Table of metadata reports.
+
+    This table has:
+
+    - One *column* per value in `ref_areas`.
+    - One *row* per metadata report in `mds`.
+    - A check-mark in cells where :func:`.contains_data_for` indicates that the metadata
+      report targets a data flow that contains data for the reference area.
+    """
+
+    template_name = "metadata-set-2.html"
+
+    #: Metadata set to summarize.
+    mds: "v21.MetadataSet"
+    #: Geographies to show.
+    ref_area: list[str]
+
+    def render(self) -> str:
+        from transport_data.org.metadata import contains_data_for
+
+        data = {
+            mdr.attaches_to.key_values["DATAFLOW"].obj.id: {  # type: ignore [union-attr]
+                ra: contains_data_for(mdr, ra) for ra in self.ref_area
+            }
+            for mdr in self.mds.report
+        }
+
+        return self.render_jinja_template(ref_area=self.ref_area, data=data)

transport_data/org/metadata/spreadsheet.py

@@ -1,3 +1,5 @@
+"""Non-standard TDC Excel file format for collecting metadata."""
+
 import logging
 import re
 from typing import TYPE_CHECKING, List, Optional, Tuple

transport_data/util/docutils.py

@@ -1,3 +1,5 @@
+"""Utilities for :mod:`docutils`."""
+
 from pathlib import Path
 from zipfile import BadZipFile, ZipFile
 

transport_data/util/hooks.py

@@ -1,3 +1,5 @@
+"""Plug-in hooks to be implemented by submodules and other packages."""
+
 from typing import TYPE_CHECKING, Iterable
 
 import pluggy

transport_data/util/jinja2.py

@@ -1,3 +1,5 @@
+"""Utilities for :mod:`jinja2`."""
+
 from functools import lru_cache
 
 

transport_data/util/pluggy.py

@@ -1,3 +1,5 @@
+"""Utilities for :mod:`pluggy`."""
+
 from importlib import import_module
 
 import pluggy

transport_data/util/pycountry.py

@@ -1,3 +1,8 @@
+"""Utilities for working with pycountry_.
+
+.. _pycountry: https://pypi.org/project/pycountry/
+"""
+
 #: Mapping from country name forms appearing in data to values recognized by
 #: :meth:`pycountry.countries.lookup`.
 NAME_MAP = {