From 0788a688a351db29310304b0c68b756b7e12f594 Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Thu, 12 Nov 2026 17:11:08 -0700 Subject: [PATCH 1/3] vinca-gha --json: write the batched stages instead of a workflow For workflows that build the stages through a matrix per stage (e.g. RoboStack's pull-request builds) instead of a generated workflow file. --- vinca/generate_gha.py | 35 ++++++++++++++++++++++++ vinca/test_generate_gha.py | 56 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index a15aee7..6fbee83 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -1,6 +1,7 @@ import argparse import glob import io +import json import os import re import sys @@ -118,6 +119,15 @@ def parse_command_line(argv: list[str]) -> Namespace: "GitHub's limit for hosted runners, 20, or 5 on macOS)", ) + parser.add_argument( + "--json", + dest="json", + default=None, + help="Write the stages to this JSON file instead of a workflow: " + '{"stages": [[{"name": job name, "recipes": "recipe ..."}, ...], ...]}, one ' + "list of jobs per stage, e.g. for a workflow's matrix", + ) + arguments = parser.parse_args(argv[1:]) config.parsed_args = arguments return arguments @@ -656,6 +666,14 @@ def main() -> None: fo.write("\n".join(order)) + if args.json: + # no workflow file, so no size limit to fit + stages = batch_stages( + filtered_stages, args.batch_size, timings=timings, followers=followers + ) + write_stages_json(stages, args.json) + return + # as many jobs as fit GitHub's workflow size limit: when the workflow is too large, # batch again with a job budget scaled down by how much it's over budget = None @@ -681,6 +699,23 @@ def main() -> None: raise RuntimeError("could not fit the workflow into GitHub's size limit") +def write_stages_json(stages: list[list[list[str]]], path: str) -> None: + """Write the batched stages as {"stages": [[job, ...], ...]}, each job + {"name": ..., "recipes": "space-separated recipes"} as the workflow jobs get them.""" + doc = { + "stages": [ + [ + {"name": get_stage_name(batch), "recipes": " ".join(batch)} + for batch in stage + ] + for stage in stages + ] + } + with open(path, "w", encoding="utf-8") as fo: + json.dump(doc, fo, indent=1) + fo.write("\n") + + def write_pipeline( args: Namespace, stages: list[list[list[str]]], diff --git a/vinca/test_generate_gha.py b/vinca/test_generate_gha.py index 68b72c1..4da5400 100644 --- a/vinca/test_generate_gha.py +++ b/vinca/test_generate_gha.py @@ -1,5 +1,6 @@ """Tests for the GitHub Actions pipeline generation.""" +import json import sys import pytest @@ -181,6 +182,61 @@ def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch): ] +def test_json_writes_the_stages_instead_of_a_workflow(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "vinca.yaml").write_text("{}\n") + recipes = {"ros2-a": [], "ros2-b": ["ros2-a"], "ros2-c": ["ros2-a"]} + for name, deps in recipes.items(): + (tmp_path / "recipes" / name).mkdir(parents=True) + recipe = { + "package": {"name": name, "version": "1.0.0"}, + "requirements": {"host": deps, "run": deps}, + } + yaml.dump(recipe, tmp_path / "recipes" / name / "recipe.yaml") + monkeypatch.setattr(generate_gha, "load_configuration", lambda: None) + monkeypatch.setattr( + sys, + "argv", + [ + "vinca-gha", + "-d", + "./recipes", + "-p", + "linux-64", + "-b", + "1", + "--json", + "stages.json", + ], + ) + + generate_gha.main() + + assert not (tmp_path / "linux.yml").exists() + stages = json.loads((tmp_path / "stages.json").read_text())["stages"] + assert [sorted(job["recipes"] for job in stage) for stage in stages] == [ + ["ros2-a"], + ["ros2-b", "ros2-c"], + ] + assert stages[0][0]["name"] == "a" + + +def test_json_without_recipes_has_no_stages(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "vinca.yaml").write_text("{}\n") + (tmp_path / "recipes").mkdir() + monkeypatch.setattr(generate_gha, "load_configuration", lambda: None) + monkeypatch.setattr( + sys, + "argv", + ["vinca-gha", "-d", "./recipes", "-p", "linux-64", "--json", "stages.json"], + ) + + generate_gha.main() + + assert json.loads((tmp_path / "stages.json").read_text()) == {"stages": []} + + @pytest.mark.parametrize( "condition,platform,outfile,needs", [ From 314d1a8b4da2937effad041751bc21f7be21c9d4 Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Thu, 12 Nov 2026 17:45:07 -0700 Subject: [PATCH 2/3] vinca-gha --json: each job's needs, the earlier jobs building its requirements --- vinca/generate_gha.py | 52 ++++++++++++++++++++++++++++---------- vinca/test_generate_gha.py | 20 +++++++++++++-- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 6fbee83..4f7362f 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -124,8 +124,9 @@ def parse_command_line(argv: list[str]) -> Namespace: dest="json", default=None, help="Write the stages to this JSON file instead of a workflow: " - '{"stages": [[{"name": job name, "recipes": "recipe ..."}, ...], ...]}, one ' - "list of jobs per stage, e.g. for a workflow's matrix", + '{"stages": [[{"id": job id, "name": job name, "recipes": "recipe ...", ' + '"needs": [job id, ...]}, ...], ...]}, one list of jobs per stage, each with ' + "the earlier jobs that build its recipes' requirements", ) arguments = parser.parse_args(argv[1:]) @@ -671,7 +672,9 @@ def main() -> None: stages = batch_stages( filtered_stages, args.batch_size, timings=timings, followers=followers ) - write_stages_json(stages, args.json) + write_stages_json( + stages, args.json, requirements if isinstance(requirements, dict) else {} + ) return # as many jobs as fit GitHub's workflow size limit: when the workflow is too large, @@ -699,18 +702,41 @@ def main() -> None: raise RuntimeError("could not fit the workflow into GitHub's size limit") -def write_stages_json(stages: list[list[list[str]]], path: str) -> None: +def write_stages_json( + stages: list[list[list[str]]], path: str, requirements: dict[str, list[str]] +) -> None: """Write the batched stages as {"stages": [[job, ...], ...]}, each job - {"name": ..., "recipes": "space-separated recipes"} as the workflow jobs get them.""" - doc = { - "stages": [ - [ - {"name": get_stage_name(batch), "recipes": " ".join(batch)} - for batch in stage - ] - for stage in stages - ] + {"id": "stage__job_", "name": ..., "recipes": "space-separated recipes", + "needs": [ids of the earlier jobs building its recipes' requirements]}: the + recipes' dependency graph as a graph of CI jobs.""" + job_of = { + recipe: f"stage_{k}_job_{i}" + for k, stage in enumerate(stages) + for i, batch in enumerate(stage) + for recipe in batch } + doc: dict[str, list[list[dict[str, Any]]]] = {"stages": []} + for k, stage in enumerate(stages): + jobs = [] + for i, batch in enumerate(stage): + job_id = f"stage_{k}_job_{i}" + needs = { + job_of[r] + for recipe in batch + for r in requirements.get(recipe, []) + if r in job_of and job_of[r] != job_id + } + jobs.append( + { + "id": job_id, + "name": get_stage_name(batch), + "recipes": " ".join(batch), + "needs": sorted( + needs, key=lambda j: [int(n) for n in j.split("_")[1::2]] + ), + } + ) + doc["stages"].append(jobs) with open(path, "w", encoding="utf-8") as fo: json.dump(doc, fo, indent=1) fo.write("\n") diff --git a/vinca/test_generate_gha.py b/vinca/test_generate_gha.py index 4da5400..e7d90a0 100644 --- a/vinca/test_generate_gha.py +++ b/vinca/test_generate_gha.py @@ -185,7 +185,13 @@ def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch): def test_json_writes_the_stages_instead_of_a_workflow(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "vinca.yaml").write_text("{}\n") - recipes = {"ros2-a": [], "ros2-b": ["ros2-a"], "ros2-c": ["ros2-a"]} + recipes = { + "ros2-a": [], + "ros2-b": ["ros2-a"], + "ros2-c": ["ros2-a"], + # ros2-published is not in ./recipes (already built): no job needed for it + "ros2-d": ["ros2-b", "ros2-published"], + } for name, deps in recipes.items(): (tmp_path / "recipes" / name).mkdir(parents=True) recipe = { @@ -217,8 +223,18 @@ def test_json_writes_the_stages_instead_of_a_workflow(tmp_path, monkeypatch): assert [sorted(job["recipes"] for job in stage) for stage in stages] == [ ["ros2-a"], ["ros2-b", "ros2-c"], + ["ros2-d"], ] - assert stages[0][0]["name"] == "a" + assert stages[0][0] == { + "id": "stage_0_job_0", + "name": "a", + "recipes": "ros2-a", + "needs": [], + } + jobs = {job["recipes"]: job for stage in stages for job in stage} + assert jobs["ros2-c"]["needs"] == ["stage_0_job_0"] + # only the jobs of its direct requirements, not ros2-a's + assert jobs["ros2-d"]["needs"] == [jobs["ros2-b"]["id"]] def test_json_without_recipes_has_no_stages(tmp_path, monkeypatch): From 0b24193a38bf053c5e8dd648fdc799fe06e0f3bd Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Fri, 13 Nov 2026 08:23:17 -0700 Subject: [PATCH 3/3] vinca-gha --json: each recipe's requirements that are built --- vinca/generate_gha.py | 19 +++++++++++++++---- vinca/test_generate_gha.py | 2 ++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 4f7362f..90b2a8d 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -125,8 +125,9 @@ def parse_command_line(argv: list[str]) -> Namespace: default=None, help="Write the stages to this JSON file instead of a workflow: " '{"stages": [[{"id": job id, "name": job name, "recipes": "recipe ...", ' - '"needs": [job id, ...]}, ...], ...]}, one list of jobs per stage, each with ' - "the earlier jobs that build its recipes' requirements", + '"needs": [job id, ...], "requires": {recipe: [recipe, ...]}}, ...], ...]}, ' + "one list of jobs per stage, each with the earlier jobs that build its " + "recipes' requirements, and each recipe's requirements that are built", ) arguments = parser.parse_args(argv[1:]) @@ -707,8 +708,10 @@ def write_stages_json( ) -> None: """Write the batched stages as {"stages": [[job, ...], ...]}, each job {"id": "stage__job_", "name": ..., "recipes": "space-separated recipes", - "needs": [ids of the earlier jobs building its recipes' requirements]}: the - recipes' dependency graph as a graph of CI jobs.""" + "needs": [ids of the earlier jobs building its recipes' requirements], + "requires": {recipe: [the recipes being built that it requires]}}: the recipes' + dependency graph as a graph of CI jobs, and the recipes' own, so that a failed + recipe can hold back only the recipes that need it.""" job_of = { recipe: f"stage_{k}_job_{i}" for k, stage in enumerate(stages) @@ -734,6 +737,14 @@ def write_stages_json( "needs": sorted( needs, key=lambda j: [int(n) for n in j.split("_")[1::2]] ), + "requires": { + recipe: sorted( + r + for r in set(requirements.get(recipe, [])) + if r in job_of and r != recipe + ) + for recipe in batch + }, } ) doc["stages"].append(jobs) diff --git a/vinca/test_generate_gha.py b/vinca/test_generate_gha.py index e7d90a0..c0bf273 100644 --- a/vinca/test_generate_gha.py +++ b/vinca/test_generate_gha.py @@ -230,11 +230,13 @@ def test_json_writes_the_stages_instead_of_a_workflow(tmp_path, monkeypatch): "name": "a", "recipes": "ros2-a", "needs": [], + "requires": {"ros2-a": []}, } jobs = {job["recipes"]: job for stage in stages for job in stage} assert jobs["ros2-c"]["needs"] == ["stage_0_job_0"] # only the jobs of its direct requirements, not ros2-a's assert jobs["ros2-d"]["needs"] == [jobs["ros2-b"]["id"]] + assert jobs["ros2-d"]["requires"] == {"ros2-d": ["ros2-b"]} def test_json_without_recipes_has_no_stages(tmp_path, monkeypatch):