vinca-gha --json: write the batched stages instead of a workflow

vinca/generate_gha.py

@@ -1,6 +1,7 @@
 import argparse
 import glob
 import io
+import json
 import os
 import re
 import sys
@@ -118,6 +119,17 @@ def parse_command_line(argv: list[str]) -> Namespace:
         "GitHub's limit for hosted runners, 20, or 5 on macOS)",
     )
 
+    parser.add_argument(
+        "--json",
+        dest="json",
+        default=None,
+        help="Write the stages to this JSON file instead of a workflow: "
+        '{"stages": [[{"id": job id, "name": job name, "recipes": "recipe ...", '
+        '"needs": [job id, ...], "requires": {recipe: [recipe, ...]}}, ...], ...]}, '
+        "one list of jobs per stage, each with the earlier jobs that build its "
+        "recipes' requirements, and each recipe's requirements that are built",
+    )
+
     arguments = parser.parse_args(argv[1:])
     config.parsed_args = arguments
     return arguments
@@ -656,6 +668,16 @@ def main() -> None:
 
         fo.write("\n".join(order))
 
+    if args.json:
+        # no workflow file, so no size limit to fit
+        stages = batch_stages(
+            filtered_stages, args.batch_size, timings=timings, followers=followers
+        )
+        write_stages_json(
+            stages, args.json, requirements if isinstance(requirements, dict) else {}
+        )
+        return
+
     # as many jobs as fit GitHub's workflow size limit: when the workflow is too large,
     # batch again with a job budget scaled down by how much it's over
     budget = None
@@ -681,6 +703,56 @@ def main() -> None:
         raise RuntimeError("could not fit the workflow into GitHub's size limit")
 
 
+def write_stages_json(
+    stages: list[list[list[str]]], path: str, requirements: dict[str, list[str]]
+) -> None:
+    """Write the batched stages as {"stages": [[job, ...], ...]}, each job
+    {"id": "stage_<k>_job_<i>", "name": ..., "recipes": "space-separated recipes",
+    "needs": [ids of the earlier jobs building its recipes' requirements],
+    "requires": {recipe: [the recipes being built that it requires]}}: the recipes'
+    dependency graph as a graph of CI jobs, and the recipes' own, so that a failed
+    recipe can hold back only the recipes that need it."""
+    job_of = {
+        recipe: f"stage_{k}_job_{i}"
+        for k, stage in enumerate(stages)
+        for i, batch in enumerate(stage)
+        for recipe in batch
+    }
+    doc: dict[str, list[list[dict[str, Any]]]] = {"stages": []}
+    for k, stage in enumerate(stages):
+        jobs = []
+        for i, batch in enumerate(stage):
+            job_id = f"stage_{k}_job_{i}"
+            needs = {
+                job_of[r]
+                for recipe in batch
+                for r in requirements.get(recipe, [])
+                if r in job_of and job_of[r] != job_id
+            }
+            jobs.append(
+                {
+                    "id": job_id,
+                    "name": get_stage_name(batch),
+                    "recipes": " ".join(batch),
+                    "needs": sorted(
+                        needs, key=lambda j: [int(n) for n in j.split("_")[1::2]]
+                    ),
+                    "requires": {
+                        recipe: sorted(
+                            r
+                            for r in set(requirements.get(recipe, []))
+                            if r in job_of and r != recipe
+                        )
+                        for recipe in batch
+                    },
+                }
+            )
+        doc["stages"].append(jobs)
+    with open(path, "w", encoding="utf-8") as fo:
+        json.dump(doc, fo, indent=1)
+        fo.write("\n")
+
+
 def write_pipeline(
     args: Namespace,
     stages: list[list[list[str]]],

vinca/test_generate_gha.py

@@ -1,5 +1,6 @@
 """Tests for the GitHub Actions pipeline generation."""
 
+import json
 import sys
 
 import pytest
@@ -181,6 +182,79 @@ def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch):
     ]
 
 
+def test_json_writes_the_stages_instead_of_a_workflow(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "vinca.yaml").write_text("{}\n")
+    recipes = {
+        "ros2-a": [],
+        "ros2-b": ["ros2-a"],
+        "ros2-c": ["ros2-a"],
+        # ros2-published is not in ./recipes (already built): no job needed for it
+        "ros2-d": ["ros2-b", "ros2-published"],
+    }
+    for name, deps in recipes.items():
+        (tmp_path / "recipes" / name).mkdir(parents=True)
+        recipe = {
+            "package": {"name": name, "version": "1.0.0"},
+            "requirements": {"host": deps, "run": deps},
+        }
+        yaml.dump(recipe, tmp_path / "recipes" / name / "recipe.yaml")
+    monkeypatch.setattr(generate_gha, "load_configuration", lambda: None)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "vinca-gha",
+            "-d",
+            "./recipes",
+            "-p",
+            "linux-64",
+            "-b",
+            "1",
+            "--json",
+            "stages.json",
+        ],
+    )
+
+    generate_gha.main()
+
+    assert not (tmp_path / "linux.yml").exists()
+    stages = json.loads((tmp_path / "stages.json").read_text())["stages"]
+    assert [sorted(job["recipes"] for job in stage) for stage in stages] == [
+        ["ros2-a"],
+        ["ros2-b", "ros2-c"],
+        ["ros2-d"],
+    ]
+    assert stages[0][0] == {
+        "id": "stage_0_job_0",
+        "name": "a",
+        "recipes": "ros2-a",
+        "needs": [],
+        "requires": {"ros2-a": []},
+    }
+    jobs = {job["recipes"]: job for stage in stages for job in stage}
+    assert jobs["ros2-c"]["needs"] == ["stage_0_job_0"]
+    # only the jobs of its direct requirements, not ros2-a's
+    assert jobs["ros2-d"]["needs"] == [jobs["ros2-b"]["id"]]
+    assert jobs["ros2-d"]["requires"] == {"ros2-d": ["ros2-b"]}
+
+
+def test_json_without_recipes_has_no_stages(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "vinca.yaml").write_text("{}\n")
+    (tmp_path / "recipes").mkdir()
+    monkeypatch.setattr(generate_gha, "load_configuration", lambda: None)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        ["vinca-gha", "-d", "./recipes", "-p", "linux-64", "--json", "stages.json"],
+    )
+
+    generate_gha.main()
+
+    assert json.loads((tmp_path / "stages.json").read_text()) == {"stages": []}
+
+
 @pytest.mark.parametrize(
     "condition,platform,outfile,needs",
     [