Expand docs of .store

Author: khaeru

doc/index.rst

@@ -77,13 +77,18 @@ The following modules contain *generic* code and utilities usable with (meta)dat
    tests
    util
 
-- :mod:`.cli`: :doc:`cli`
+- :mod:`~transport_data.cli`: :doc:`cli`
 - :mod:`.config`: :doc:`config`
 - :mod:`.store`: :doc:`store`
 - :mod:`.testing`: :doc:`testing`
 - :mod:`.tests`: :doc:`tests`
 - :mod:`.util`: :doc:`util`
 
+There are also the following top-level objects:
+
+.. autodata:: transport_data.CONFIG
+.. autodata:: transport_data.STORE
+
 Indices and tables
 ==================
 

doc/store.rst

@@ -1,4 +1,54 @@
 (Meta)data storage
 ******************
 
+:mod:`transport_data.store` provides for storing and handling two kinds of data files:
+
+Local data
+   Data that exist only on an individual user's system.
+   These may include:
+
+   - Cached files such as those downloaded from particular network locations.
+   - Temporary or intermediate data files produced or used by other code in :mod:`transport_data` or downstream packages.
+
+   These data are handled by an instance of the :class:`.LocalStore` class.
+
+Maintained data
+   These are data that may be distributed and referenced by other users and SDMX data or structures.
+   They are handled by an instance of the :class:`.Registry` class.
+
+:data:`transport_data.STORE` is an instance of the :class:`.UnionStore` class, which dispatches operations to :class:`.LocalStore` or :class:`.Registry` according to the maintainer of objects.
+All three classes are subclassed of :class:`.BaseStore` and share a common interface.
+
+.. _store-layout:
+
+Formats and layout
+==================
+
+- Files are stored in the (configurable) directory indicated by :attr:`.Config.data_path`.
+  This defaults to the :ref:`platformdirs:api:user data directory`.
+  For instance, on a Unix/Linux system, the data may be in :file:`$HOME/.local/share/transport-data/`.
+
+  - :class:`.LocalStore` files are in a :file:`…/local/` subdirectory.
+  - :class:`.Registry` files are in a :file:`…/registry/` subdirectory.
+
+- All files are stored as SDMX-ML (XML).
+
+  Every SDMX :class:`~sdmx.model.common.MaintainableArtefact` has a `uniform resource name (URN) <https://en.wikipedia.org/wiki/Uniform_Resource_Name>`__ that resembles ``urn:sdmx:org.sdmx.infomodel.codelist.Codelist=TEST:COLOUR(1.2.3)``.
+  In this example:
+
+  - "TEST" is the :attr:`~sdmx.model.common.IdentifiableArtefact.id` of a :class:`~sdmx.model.common.Agency` stored as the :attr:`~sdmx.model.common.MaintainableArtefact.maintainer` attribute.
+  - "COLOUR" is the ID of the object itself.
+  - "(1.2.3)" is the :attr:`~sdmx.model.common.VersionableArtefact.version` attribute of a :class:`~sdmx.model.common.VersionableArtefact`.
+
+- Directory and file names include maintainer ID, object class name, and object ID.
+  For instance, the file :file:`TEST/Codelist_TEST_COLOUR.xml` contains the :class:`~sdmx.model.common.Codelist` with ID "COLOUR" maintained by the agency with ID "TEST".
+- Each file contains **only** objects of the class, ID, and maintainer corresponding to its path.
+- Each file **may** contain *multiple versions* of objects.
+  For instance, the file :file:`TEST/Codelist_TEST_COLOUR.xml` may contain objects with the URNs and versions:
+
+  - ``urn:sdmx:org.sdmx.infomodel.codelist.Codelist=TEST:COLOUR(1.0.0)`` → version 1.0.0;
+  - ``urn:sdmx:org.sdmx.infomodel.codelist.Codelist=TEST:COLOUR(1.2.3)`` → version 1.2.3;
+  - ``urn:sdmx:org.sdmx.infomodel.codelist.Codelist=TEST:COLOUR(1.2.4)`` → version 1.2.4;
+  - and so on.
+
 .. include:: _api/transport_data.store.rst

transport_data/__init__.py

@@ -8,5 +8,8 @@
 log.setLevel(logging.INFO)
 log.addHandler(logging.StreamHandler(sys.stdout))
 
+#: Global configuration.
 CONFIG = Config.read()
+
+#: Global access to data storage.
 STORE = UnionStore(CONFIG)

transport_data/config.py

@@ -11,10 +11,10 @@
 class Config:
     """Common configuration."""
 
-    #: Path to the configuration file read, if any
+    #: Path to the configuration file read, if any.
     config_path: Optional[Path] = None
 
-    #: Path to a local clone of https://github.com/transport-data/registry.
+    #: Path for local data. See :ref:`store-layout` for details.
     data_path: Path = field(
         default_factory=lambda: user_data_path("transport-data", ensure_exists=True)
     )
@@ -32,6 +32,7 @@ def _config_path():
 
     @classmethod
     def read(cls):
+        """Read configuration from file, returning a new instance."""
         cp = cls._config_path()
         if cp.exists():
             with open(cp) as f:
@@ -44,6 +45,7 @@ def read(cls):
         return cls(**data)
 
     def write(self):
+        """Write configuration to file."""
         cp = self._config_path()
         if not cp.parent.exists():
             print(f"Create {cp.parent}")

transport_data/store.py

@@ -46,6 +46,7 @@ def _maintainer(obj: Union[m.MaintainableArtefact, m.DataSet]) -> m.Agency:
 class BaseStore(ABC):
     """A class for file-based storage of SDMX artefacts."""
 
+    #: Top level file system path containing stored files.
     path: Path
 
     @abstractmethod
@@ -203,7 +204,17 @@ def __init__(self, config) -> None:
 
 
 class Registry(BaseStore):
-    """The transport-data/registry Git repository."""
+    """Registry of TDCI-maintained structure information.
+
+    Currently these are stored in, and available from, the
+    `transport-data/registry <https://github.com/transport-data/registry>`_ GitHub
+    repository. This class handles the interaction with a local clone of that
+    repository.
+
+    In the future, they will be maintained on a full SDMX REST web service, and this
+    class will handle retrieving from and publishing to that web service, with
+    appropriate caching etc.
+    """
 
     def __init__(self, config) -> None:
         self.path = config.data_path.joinpath("registry")
@@ -251,6 +262,7 @@ def write(
 class UnionStore(BaseStore):
     """A store that maps between a :class:`.LocalStore` and :class:`.Registry`."""
 
+    #: Instances of :class:`.LocalStore` and :class:`.Registry`.
     store: Mapping[Literal["local", "registry"], BaseStore]
 
     #: Mapping from maintainer IDs (such as "TEST" or "TDCI") to either "local" or
@@ -294,6 +306,7 @@ def assign_version(
         )
 
     def clone(self):
+        """Clone the underlying :class:`.Registry`."""
         self.store["registry"].clone()