tests: Add fail_on_changes to toxgen (#4072)

Add `fail_on_changes` to toxgen. The idea is that the script will now
have two modes:

- **Normal mode** (when `fail_on_changes` is `False`) that is used to
actually generate the `tox.ini` file. This [will
be](#4050) run in a
cron job in CI and create a PR with the updated test setup.
- The newly added **fail-on-changes mode** (when `fail_on_changes` is
`True`) that is used to detect manual changes to one of the affected
files without updating the rest (e.g. making a manual change to
`tox.ini` without updating the `tox.jinja` template). This will be run
in CI similar to the `fail_on_changes` check of `split-tox-gh-actions`.

The problem with detecting manual changes is that if we just reran the
script on each PR, chances are it would pull in new releases that are
not part of the `tox.ini` on master, making the file look different from
what was committed as if it had unrelated manual changes.

To counteract this, we now store the timestamp when the file was last
generated in `tox.ini`. We use this in fail-on-changes mode to filter
out releases that popped up after the file was last generated. This way,
the package versions should be the same and if there is anything
different in `tox.ini`, it's likely to be the manual changes that we
want to detect.

Closes #4051

Author: sentrivana

.github/workflows/ci.yml

@@ -44,7 +44,11 @@ jobs:
         with:
           python-version: 3.12
 
-      - run: |
+      - name: Detect unexpected changes to tox.ini or CI
+        run: |
+          pip install -e .
+          pip install -r scripts/populate_tox/requirements.txt
+          python scripts/populate_tox/populate_tox.py --fail-on-changes
           pip install -r scripts/split_tox_gh_actions/requirements.txt
           python scripts/split_tox_gh_actions/split_tox_gh_actions.py --fail-on-changes
 

scripts/populate_tox/populate_tox.py

@@ -3,15 +3,18 @@
 """
 
 import functools
+import hashlib
 import os
 import sys
 import time
 from bisect import bisect_left
 from collections import defaultdict
+from datetime import datetime, timezone
 from importlib.metadata import metadata
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
 from pathlib import Path
+from textwrap import dedent
 from typing import Optional, Union
 
 # Adding the scripts directory to PATH. This is necessary in order to be able
@@ -106,7 +109,9 @@ def fetch_release(package: str, version: Version) -> dict:
     return pypi_data.json()
 
 
-def _prefilter_releases(integration: str, releases: dict[str, dict]) -> list[Version]:
+def _prefilter_releases(
+    integration: str, releases: dict[str, dict], older_than: Optional[datetime] = None
+) -> list[Version]:
     """
     Filter `releases`, removing releases that are for sure unsupported.
 
@@ -135,6 +140,10 @@ def _prefilter_releases(integration: str, releases: dict[str, dict]) -> list[Ver
         if meta["yanked"]:
             continue
 
+        if older_than is not None:
+            if datetime.fromisoformat(meta["upload_time_iso_8601"]) > older_than:
+                continue
+
         version = Version(release)
 
         if min_supported and version < min_supported:
@@ -160,19 +169,24 @@ def _prefilter_releases(integration: str, releases: dict[str, dict]) -> list[Ver
     return sorted(filtered_releases)
 
 
-def get_supported_releases(integration: str, pypi_data: dict) -> list[Version]:
+def get_supported_releases(
+    integration: str, pypi_data: dict, older_than: Optional[datetime] = None
+) -> list[Version]:
     """
     Get a list of releases that are currently supported by the SDK.
 
     This takes into account a handful of parameters (Python support, the lowest
     version we've defined for the framework, the date of the release).
+
+    If an `older_than` timestamp is provided, no release newer than that will be
+    considered.
     """
     package = pypi_data["info"]["name"]
 
     # Get a consolidated list without taking into account Python support yet
     # (because that might require an additional API call for some
     # of the releases)
-    releases = _prefilter_releases(integration, pypi_data["releases"])
+    releases = _prefilter_releases(integration, pypi_data["releases"], older_than)
 
     # Determine Python support
     expected_python_versions = TEST_SUITE_CONFIG[integration].get("python")
@@ -381,7 +395,9 @@ def _render_dependencies(integration: str, releases: list[Version]) -> list[str]
     return rendered
 
 
-def write_tox_file(packages: dict) -> None:
+def write_tox_file(
+    packages: dict, update_timestamp: bool, last_updated: datetime
+) -> None:
     template = ENV.get_template("tox.jinja")
 
     context = {"groups": {}}
@@ -400,6 +416,11 @@ def write_tox_file(packages: dict) -> None:
                 }
             )
 
+    if update_timestamp:
+        context["updated"] = datetime.now(tz=timezone.utc).isoformat()
+    else:
+        context["updated"] = last_updated.isoformat()
+
     rendered = template.render(context)
 
     with open(TOX_FILE, "w") as file:
@@ -453,7 +474,59 @@ def _add_python_versions_to_release(
     release.rendered_python_versions = _render_python_versions(release.python_versions)
 
 
-def main() -> None:
+def get_file_hash() -> str:
+    """Calculate a hash of the tox.ini file."""
+    hasher = hashlib.md5()
+
+    with open(TOX_FILE, "rb") as f:
+        buf = f.read()
+        hasher.update(buf)
+
+    return hasher.hexdigest()
+
+
+def get_last_updated() -> Optional[datetime]:
+    timestamp = None
+
+    with open(TOX_FILE, "r") as f:
+        for line in f:
+            if line.startswith("# Last generated:"):
+                timestamp = datetime.fromisoformat(line.strip().split()[-1])
+                break
+
+    if timestamp is None:
+        print(
+            "Failed to find out when tox.ini was last generated; the timestamp seems to be missing from the file."
+        )
+
+    return timestamp
+
+
+def main(fail_on_changes: bool = False) -> None:
+    """
+    Generate tox.ini from the tox.jinja template.
+
+    The script has two modes of operation:
+    - fail on changes mode (if `fail_on_changes` is True)
+    - normal mode (if `fail_on_changes` is False)
+
+    Fail on changes mode is run on every PR to make sure that `tox.ini`,
+    `tox.jinja` and this script don't go out of sync because of manual changes
+    in one place but not the other.
+
+    Normal mode is meant to be run as a cron job, regenerating tox.ini and
+    proposing the changes via a PR.
+    """
+    print(f"Running in {'fail_on_changes' if fail_on_changes else 'normal'} mode.")
+    last_updated = get_last_updated()
+    if fail_on_changes:
+        # We need to make the script ignore any new releases after the `last_updated`
+        # timestamp so that we don't fail CI on a PR just because a new package
+        # version was released, leading to unrelated changes in tox.ini.
+        print(
+            f"Since we're in fail_on_changes mode, we're only considering releases before the last tox.ini update at {last_updated.isoformat()}."
+        )
+
     global MIN_PYTHON_VERSION, MAX_PYTHON_VERSION
     sdk_python_versions = _parse_python_versions_from_classifiers(
         metadata("sentry-sdk").get_all("Classifier")
@@ -480,7 +553,9 @@ def main() -> None:
             pypi_data = fetch_package(package)
 
             # Get the list of all supported releases
-            releases = get_supported_releases(integration, pypi_data)
+            # If in check mode, ignore releases newer than `last_updated`
+            older_than = last_updated if fail_on_changes else None
+            releases = get_supported_releases(integration, pypi_data, older_than)
             if not releases:
                 print("  Found no supported releases.")
                 continue
@@ -510,8 +585,44 @@ def main() -> None:
                     }
                 )
 
-    write_tox_file(packages)
+    if fail_on_changes:
+        old_file_hash = get_file_hash()
+
+    write_tox_file(
+        packages, update_timestamp=not fail_on_changes, last_updated=last_updated
+    )
+
+    if fail_on_changes:
+        new_file_hash = get_file_hash()
+        if old_file_hash != new_file_hash:
+            raise RuntimeError(
+                dedent(
+                    """
+                Detected that `tox.ini` is out of sync with
+                `scripts/populate_tox/tox.jinja` and/or
+                `scripts/populate_tox/populate_tox.py`. This might either mean
+                that `tox.ini` was changed manually, or the `tox.jinja`
+                template and/or the `populate_tox.py` script were changed without
+                regenerating `tox.ini`.
+
+                Please don't make manual changes to `tox.ini`. Instead, make the
+                changes to the `tox.jinja` template and/or the `populate_tox.py`
+                script (as applicable) and regenerate the `tox.ini` file with:
+
+                python -m venv toxgen.env
+                . toxgen.env/bin/activate
+                pip install -r scripts/populate_tox/requirements.txt
+                python scripts/populate_tox/populate_tox.py
+                """
+                )
+            )
+        print("Done checking tox.ini. Looking good!")
+    else:
+        print(
+            "Done generating tox.ini. Make sure to also update the CI YAML files to reflect the new test targets."
+        )
 
 
 if __name__ == "__main__":
-    main()
+    fail_on_changes = len(sys.argv) == 2 and sys.argv[1] == "--fail-on-changes"
+    main(fail_on_changes)

scripts/populate_tox/tox.jinja

@@ -9,6 +9,8 @@
 # or in the script (if you want to change the auto-generated part).
 # The file (and all resulting CI YAMLs) then need to be regenerated via
 # "scripts/generate-test-files.sh".
+#
+# Last generated: {{ updated }}
 
 [tox]
 requires =

tox.ini

@@ -9,6 +9,8 @@
 # or in the script (if you want to change the auto-generated part).
 # The file (and all resulting CI YAMLs) then need to be regenerated via
 # "scripts/generate-test-files.sh".
+#
+# Last generated: 2025-02-18T12:57:32.874168+00:00
 
 [tox]
 requires =
@@ -290,7 +292,7 @@ envlist =
     {py3.6,py3.7,py3.8}-trytond-v5.8.16
     {py3.8,py3.10,py3.11}-trytond-v6.8.17
     {py3.8,py3.11,py3.12}-trytond-v7.0.9
-    {py3.8,py3.11,py3.12}-trytond-v7.4.5
+    {py3.8,py3.11,py3.12}-trytond-v7.4.6
 
     {py3.7,py3.11,py3.12}-typer-v0.15.1
 
@@ -714,7 +716,7 @@ deps =
     trytond-v5.8.16: trytond==5.8.16
     trytond-v6.8.17: trytond==6.8.17
     trytond-v7.0.9: trytond==7.0.9
-    trytond-v7.4.5: trytond==7.4.5
+    trytond-v7.4.6: trytond==7.4.6
     trytond: werkzeug
     trytond-v4.6.9: werkzeug<1.0
     trytond-v4.8.18: werkzeug<1.0