diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index a15aee7..90b2a8d 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -1,6 +1,7 @@ import argparse import glob import io +import json import os import re import sys @@ -118,6 +119,17 @@ def parse_command_line(argv: list[str]) -> Namespace: "GitHub's limit for hosted runners, 20, or 5 on macOS)", ) + parser.add_argument( + "--json", + dest="json", + default=None, + help="Write the stages to this JSON file instead of a workflow: " + '{"stages": [[{"id": job id, "name": job name, "recipes": "recipe ...", ' + '"needs": [job id, ...], "requires": {recipe: [recipe, ...]}}, ...], ...]}, ' + "one list of jobs per stage, each with the earlier jobs that build its " + "recipes' requirements, and each recipe's requirements that are built", + ) + arguments = parser.parse_args(argv[1:]) config.parsed_args = arguments return arguments @@ -656,6 +668,16 @@ def main() -> None: fo.write("\n".join(order)) + if args.json: + # no workflow file, so no size limit to fit + stages = batch_stages( + filtered_stages, args.batch_size, timings=timings, followers=followers + ) + write_stages_json( + stages, args.json, requirements if isinstance(requirements, dict) else {} + ) + return + # as many jobs as fit GitHub's workflow size limit: when the workflow is too large, # batch again with a job budget scaled down by how much it's over budget = None @@ -681,6 +703,56 @@ def main() -> None: raise RuntimeError("could not fit the workflow into GitHub's size limit") +def write_stages_json( + stages: list[list[list[str]]], path: str, requirements: dict[str, list[str]] +) -> None: + """Write the batched stages as {"stages": [[job, ...], ...]}, each job + {"id": "stage__job_", "name": ..., "recipes": "space-separated recipes", + "needs": [ids of the earlier jobs building its recipes' requirements], + "requires": {recipe: [the recipes being built that it requires]}}: the recipes' + dependency graph as a graph of CI jobs, and the recipes' own, so that a failed + recipe can hold back only the recipes that need it.""" + job_of = { + recipe: f"stage_{k}_job_{i}" + for k, stage in enumerate(stages) + for i, batch in enumerate(stage) + for recipe in batch + } + doc: dict[str, list[list[dict[str, Any]]]] = {"stages": []} + for k, stage in enumerate(stages): + jobs = [] + for i, batch in enumerate(stage): + job_id = f"stage_{k}_job_{i}" + needs = { + job_of[r] + for recipe in batch + for r in requirements.get(recipe, []) + if r in job_of and job_of[r] != job_id + } + jobs.append( + { + "id": job_id, + "name": get_stage_name(batch), + "recipes": " ".join(batch), + "needs": sorted( + needs, key=lambda j: [int(n) for n in j.split("_")[1::2]] + ), + "requires": { + recipe: sorted( + r + for r in set(requirements.get(recipe, [])) + if r in job_of and r != recipe + ) + for recipe in batch + }, + } + ) + doc["stages"].append(jobs) + with open(path, "w", encoding="utf-8") as fo: + json.dump(doc, fo, indent=1) + fo.write("\n") + + def write_pipeline( args: Namespace, stages: list[list[list[str]]], diff --git a/vinca/test_generate_gha.py b/vinca/test_generate_gha.py index 68b72c1..c0bf273 100644 --- a/vinca/test_generate_gha.py +++ b/vinca/test_generate_gha.py @@ -1,5 +1,6 @@ """Tests for the GitHub Actions pipeline generation.""" +import json import sys import pytest @@ -181,6 +182,79 @@ def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch): ] +def test_json_writes_the_stages_instead_of_a_workflow(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "vinca.yaml").write_text("{}\n") + recipes = { + "ros2-a": [], + "ros2-b": ["ros2-a"], + "ros2-c": ["ros2-a"], + # ros2-published is not in ./recipes (already built): no job needed for it + "ros2-d": ["ros2-b", "ros2-published"], + } + for name, deps in recipes.items(): + (tmp_path / "recipes" / name).mkdir(parents=True) + recipe = { + "package": {"name": name, "version": "1.0.0"}, + "requirements": {"host": deps, "run": deps}, + } + yaml.dump(recipe, tmp_path / "recipes" / name / "recipe.yaml") + monkeypatch.setattr(generate_gha, "load_configuration", lambda: None) + monkeypatch.setattr( + sys, + "argv", + [ + "vinca-gha", + "-d", + "./recipes", + "-p", + "linux-64", + "-b", + "1", + "--json", + "stages.json", + ], + ) + + generate_gha.main() + + assert not (tmp_path / "linux.yml").exists() + stages = json.loads((tmp_path / "stages.json").read_text())["stages"] + assert [sorted(job["recipes"] for job in stage) for stage in stages] == [ + ["ros2-a"], + ["ros2-b", "ros2-c"], + ["ros2-d"], + ] + assert stages[0][0] == { + "id": "stage_0_job_0", + "name": "a", + "recipes": "ros2-a", + "needs": [], + "requires": {"ros2-a": []}, + } + jobs = {job["recipes"]: job for stage in stages for job in stage} + assert jobs["ros2-c"]["needs"] == ["stage_0_job_0"] + # only the jobs of its direct requirements, not ros2-a's + assert jobs["ros2-d"]["needs"] == [jobs["ros2-b"]["id"]] + assert jobs["ros2-d"]["requires"] == {"ros2-d": ["ros2-b"]} + + +def test_json_without_recipes_has_no_stages(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "vinca.yaml").write_text("{}\n") + (tmp_path / "recipes").mkdir() + monkeypatch.setattr(generate_gha, "load_configuration", lambda: None) + monkeypatch.setattr( + sys, + "argv", + ["vinca-gha", "-d", "./recipes", "-p", "linux-64", "--json", "stages.json"], + ) + + generate_gha.main() + + assert json.loads((tmp_path / "stages.json").read_text()) == {"stages": []} + + @pytest.mark.parametrize( "condition,platform,outfile,needs", [