Generate with separate files

Author: ocelotl

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py

@@ -37,7 +37,7 @@ def get_tox_envs(tox_ini_path: Path) -> list:
     return core_config_set.load("env_list")
 
 
-def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list:
+def get_test_job_datas(tox_envs: list, operating_systems: list) -> list:
 
     os_alias = {
         "ubuntu-latest": "Ubuntu",
@@ -53,7 +53,7 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list:
         "py312": "3.12",
     }
 
-    test_jobs_data = []
+    test_job_datas = []
 
     tox_test_env_regex = re_compile(
         r"(?P<python_version>py\w+)-test-"
@@ -83,7 +83,7 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list:
             else:
                 test_requirements = f"-{test_requirements} "
 
-            test_jobs_data.append(
+            test_job_datas.append(
                 {
                     "name": f"{tox_env}_{operating_system}",
                     "ui_name": (
@@ -99,12 +99,12 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list:
 
             )
 
-    return test_jobs_data
+    return test_job_datas
 
 
-def get_lint_jobs_data(tox_envs: list) -> list:
+def get_lint_job_datas(tox_envs: list) -> list:
 
-    lint_jobs_data = []
+    lint_job_datas = []
 
     tox_lint_env_regex = re_compile(r"lint-(?P<name>[-\w]+)")
 
@@ -117,7 +117,7 @@ def get_lint_jobs_data(tox_envs: list) -> list:
 
         tox_env = tox_lint_env_match.string
 
-        lint_jobs_data.append(
+        lint_job_datas.append(
             {
                 "name": f"{tox_env}",
                 "ui_name": f"{tox_lint_env_match.groupdict()['name']}",
@@ -126,12 +126,12 @@ def get_lint_jobs_data(tox_envs: list) -> list:
 
         )
 
-    return lint_jobs_data
+    return lint_job_datas
 
 
-def get_contrib_jobs_data(tox_envs: list) -> list:
+def get_contrib_job_datas(tox_envs: list) -> list:
 
-    contrib_jobs_data = []
+    contrib_job_datas = []
 
     tox_contrib_env_regex = re_compile(
         r"py38-test-(?P<name>[-\w]+\w)-?(?P<contrib_requirements>\d+)?"
@@ -156,7 +156,7 @@ def get_contrib_jobs_data(tox_envs: list) -> list:
         else:
             contrib_requirements = f"-{contrib_requirements} "
 
-        contrib_jobs_data.append(
+        contrib_job_datas.append(
             {
                 "ui_name": (
                     f"{groups['name']}"
@@ -167,7 +167,35 @@ def get_contrib_jobs_data(tox_envs: list) -> list:
 
         )
 
-    return contrib_jobs_data
+    return contrib_job_datas
+
+
+def _generate_workflow(
+    job_datas: list, name: str, workflow_directory_path: Path
+):
+
+    # Github seems to limit the amount of jobs in a workflow file, that is why
+    # they are split in groups of 250 per workflow file.
+    for file_number, job_datas in enumerate(
+        [
+            job_datas[index:index + 250]
+            for index in range(0, len(job_datas), 250)
+        ]
+    ):
+
+        with open(
+            workflow_directory_path.joinpath(f"{name}_{file_number}.yml"),
+            "w"
+        ) as test_yml_file:
+
+            test_yml_file.write(
+                Environment(
+                    loader=FileSystemLoader(Path(__file__).parent)
+                ).get_template(f"{name}.yml.j2").render(
+                    job_datas=job_datas, file_number=file_number
+                )
+            )
+            test_yml_file.write("\n")
 
 
 def generate_test_workflow(
@@ -176,69 +204,33 @@ def generate_test_workflow(
     *operating_systems
 ) -> None:
 
-    jobs = get_test_jobs_data(get_tox_envs(tox_ini_path), operating_systems)
-
-    with (
-        open(workflow_directory_path.joinpath("test_0.yml"), "w") as
-        test_yml_file
-    ):
-        test_yml_file.write(
-            Environment(
-                loader=FileSystemLoader(Path(__file__).parent)
-            ).get_template("test.yml.j2").render(
-                jobs=jobs[:250], file_number=0
-            )
-        )
-        test_yml_file.write("\n")
-
-    with (
-        open(workflow_directory_path.joinpath("test_1.yml"), "w") as
-        test_yml_file
-    ):
-        test_yml_file.write(
-            Environment(
-                loader=FileSystemLoader(Path(__file__).parent)
-            ).get_template("test.yml.j2").render(
-                jobs=jobs[250:], file_number=1
-            )
-        )
-        test_yml_file.write("\n")
+    _generate_workflow(
+        get_test_job_datas(get_tox_envs(tox_ini_path), operating_systems),
+        "test",
+        workflow_directory_path
+    )
 
 
 def generate_lint_workflow(
     tox_ini_path: Path,
     workflow_directory_path: Path,
 ) -> None:
 
-    with (
-        open(workflow_directory_path.joinpath("lint.yml"), "w") as
-        lint_yml_file
-    ):
-        lint_yml_file.write(
-            Environment(
-                loader=FileSystemLoader(Path(__file__).parent)
-            ).get_template("lint.yml.j2").render(
-                jobs=get_lint_jobs_data(get_tox_envs(tox_ini_path))
-            )
-        )
-        lint_yml_file.write("\n")
+    _generate_workflow(
+        get_lint_job_datas(get_tox_envs(tox_ini_path)),
+        "lint",
+        workflow_directory_path
+    )
 
 
 def generate_contrib_workflow(
     workflow_directory_path: Path,
 ) -> None:
 
-    with (
-        open(workflow_directory_path.joinpath("contrib.yml"), "w") as
-        contrib_yml_file
-    ):
-        contrib_yml_file.write(
-            Environment(
-                loader=FileSystemLoader(Path(__file__).parent)
-            ).get_template("contrib.yml.j2").render(
-                jobs=get_contrib_jobs_data(
-                    get_tox_envs(Path(__file__).parent.joinpath("tox.ini"))
-                )
-            )
-        )
-        contrib_yml_file.write("\n")
+    _generate_workflow(
+        get_contrib_job_datas(
+            get_tox_envs(Path(__file__).parent.joinpath("tox.ini"))
+        ),
+        "contrib",
+        workflow_directory_path
+    )

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/contrib.yml.j2

@@ -1,7 +1,7 @@
 # Do not edit this file.
 # This file is generated automatically by executing tox -e generate_workflows
 
-name: Contrib
+name: Contrib {{ file_number }}
 
 on:
   push:
@@ -14,10 +14,10 @@ env:
   PIP_EXISTS_ACTION: w
 
 jobs:
-  {%- for job in jobs %}
+  {%- for job_data in job_datas %}
 
-  {{ job.tox_env }}:
-    name: {{ job.ui_name }}
+  {{ job_data.tox_env }}:
+    name: {{ job_data.ui_name }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout contrib repo @ SHA - ${% raw %}{{ env.CONTRIB_REPO_SHA }}{% endraw %}
@@ -42,5 +42,5 @@ jobs:
         run: pip install tox
 
       - name: Run tests
-        run: tox -e {{ job.tox_env }} -- -ra
+        run: tox -e {{ job_data.tox_env }} -- -ra
   {%- endfor %}

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2

@@ -1,7 +1,7 @@
 # Do not edit this file.
 # This file is generated automatically by executing tox -e generate_workflows
 
-name: Lint
+name: Lint {{ file_number }}
 
 on:
   push:
@@ -12,10 +12,10 @@ env:
   PIP_EXISTS_ACTION: w
 
 jobs:
-  {%- for job in jobs %}
+  {%- for job_data in job_datas %}
 
-  {{ job.name }}:
-    name: {{ job.ui_name }}
+  {{ job_data.name }}:
+    name: {{ job_data.ui_name }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
@@ -30,5 +30,5 @@ jobs:
         run: pip install tox
 
       - name: Run tests
-        run: tox -e {{ job.tox_env }}
+        run: tox -e {{ job_data.tox_env }}
   {%- endfor %}

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2

@@ -12,28 +12,28 @@ env:
   PIP_EXISTS_ACTION: w
 
 jobs:
-  {%- for job in jobs %}
+  {%- for job_data in job_datas %}
 
-  {{ job.name }}:
-    name: {{ job.ui_name }}
-    runs-on: {{ job.os }}
+  {{ job_data.name }}:
+    name: {{ job_data.ui_name }}
+    runs-on: {{ job_data.os }}
     steps:
       - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
         uses: actions/checkout@v4
 
-      - name: Set up Python {{ job.python_version }}
+      - name: Set up Python {{ job_data.python_version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "{{ job.python_version }}"
+          python-version: "{{ job_data.python_version }}"
 
       - name: Install tox
         run: pip install tox
-      {%- if job.os == "windows-latest" %}
+      {%- if job_data.os == "windows-latest" %}
 
       - name: Configure git to support long filenames
         run: git config --system core.longpaths true
       {%- endif %}
 
       - name: Run tests
-        run: tox -e {{ job.tox_env }} -- -ra
+        run: tox -e {{ job_data.tox_env }} -- -ra
   {%- endfor %}

.github/workflows/lint.yml .github/workflows/lint_0.yml

@@ -1,7 +1,7 @@
 # Do not edit this file.
 # This file is generated automatically by executing tox -e generate_workflows
 
-name: Lint
+name: Lint 0
 
 on:
   push: