From bc76b30b47613037ee6745cfb7487bea57ea6692 Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Mon, 2 Nov 2026 16:29:20 +0100 Subject: [PATCH] Build only changed releases during metadata updates --- .github/workflows/docker-build.yml | 26 +++++- .github/workflows/update.yml | 6 ++ README.md | 6 ++ build.py | 51 +++++++--- tests/test_changed_builds.py | 144 +++++++++++++++++++++++++++++ 5 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 tests/test_changed_builds.py diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index b715564c..78244bfb 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -6,7 +6,18 @@ on: - master paths: - buildinfo.json + - build.py + - 'docker/**' workflow_call: + inputs: + previous_sha: + type: string + required: false + default: '' + checkout_ref: + type: string + required: false + default: '' # workaround for #526 workflow_dispatch: @@ -18,15 +29,24 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: - ref: ${{ github.ref_name }} + ref: ${{ inputs.checkout_ref || github.sha }} + fetch-depth: 0 - name: Set up QEMU uses: docker/setup-qemu-action@v4 - - name: build and push all images + - name: Build and push changed images if: ${{ env.DOCKER_USERNAME != '' && env.DOCKER_PASSWORD != '' }} env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + PREVIOUS_SHA: ${{ inputs.previous_sha || github.event.before }} run: | - ./build.py --push-tags --multiarch --both + args=(--push-tags --multiarch --both) + # Runtime/build changes and manual runs rebuild all current versions. + if [[ -n "$PREVIOUS_SHA" ]] && git cat-file -e "$PREVIOUS_SHA^{commit}" 2>/dev/null; then + if git diff --quiet "$PREVIOUS_SHA" HEAD -- docker build.py; then + args+=(--changed-from "$PREVIOUS_SHA") + fi + fi + ./build.py "${args[@]}" diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 2b373e2c..af906f17 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -11,6 +11,8 @@ jobs: if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'factoriotools/factorio-docker') outputs: updated: ${{ steps.changes.outputs.updated }} + before_sha: ${{ steps.before.outputs.sha }} + head_sha: ${{ steps.changes.outputs.sha }} steps: - name: Checkout @@ -27,6 +29,7 @@ jobs: - name: Check for changes id: changes run: | + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" if [ "$(git rev-parse HEAD)" != "${{ steps.before.outputs.sha }}" ]; then echo "updated=true" >> "$GITHUB_OUTPUT" else @@ -37,4 +40,7 @@ jobs: needs: check if: needs.check.outputs.updated == 'true' uses: ./.github/workflows/docker-build.yml + with: + previous_sha: ${{ needs.check.outputs.before_sha }} + checkout_ref: ${{ needs.check.outputs.head_sha }} secrets: inherit diff --git a/README.md b/README.md index 83f35ae4..c4cb77db 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,12 @@ The `PRESET` environment variable is used when generating a new map (when `GENER If PRESET is not specified or left empty, the map will be generated using the settings from `map-gen-settings.json` and `map-settings.json` without a preset. +## Building images + +Normal deployments should pull published images rather than build them. For development, run `python3 build.py --both` from the repository root to build regular and rootless variants. Add `--multiarch --push-tags` only when you intend to publish images and have registry credentials configured. + +`--changed-from GIT_REF` compares release metadata with an earlier commit and builds only new versions, changed archive checksums, or versions gaining tags. Removing an alias alone does not rebuild its previous owner. The scheduled updater passes its before/after commit IDs so an experimental-only release does not republish unchanged stable images. Runtime/build-source changes and manually dispatched builds still rebuild all current versions. To retry a failed publication or deliberately refresh base layers, run a full build without `--changed-from`. + ## Container Details The philosophy is to [keep it simple](http://wiki.c2.com/?KeepItSimple). diff --git a/build.py b/build.py index 737fc1f9..72874375 100755 --- a/build.py +++ b/build.py @@ -99,6 +99,27 @@ def generate_rootless_tags(original_tags): return [f"{tag}-rootless" for tag in original_tags] +def load_previous_buildinfo(ref): + """Read release metadata at a git ref without changing the checkout.""" + contents = subprocess.check_output( + ["git", "show", "--end-of-options", f"{ref}:buildinfo.json"], + cwd=os.path.dirname(os.path.abspath(__file__)), text=True, + ) + data = json.loads(contents) + if not isinstance(data, dict): + raise ValueError("Previous buildinfo.json must be an object") + return data + + +def release_changed(version, info, previous): + old = previous.get(version) + if old is None: + return True + # Removing an alias needs no rebuild: the new owner will publish that tag. + return (any(info.get(key) != old.get(key) for key in ("sha256", "sha256_arm64")) + or bool(set(info["tags"]) - set(old["tags"]))) + + def main(): parser = argparse.ArgumentParser(description='Build Factorio Docker images') parser.add_argument('--push-tags', action='store_true', help='Push images to Docker Hub') @@ -108,6 +129,8 @@ def main(): parser.add_argument('--only-stable-latest', action='store_true', help='Build only stable and latest versions (for rootless by default)') + parser.add_argument('--changed-from', metavar='GIT_REF', + help='Build only versions with new archives or added tags since this ref') args = parser.parse_args() # Default behavior: build regular images unless specified otherwise @@ -117,12 +140,18 @@ def main(): with open(os.path.join(os.path.dirname(__file__), "buildinfo.json")) as file_handle: builddata = json.load(file_handle) - if args.push_tags: - login() + previous = None + if args.changed_from: + try: + previous = load_previous_buildinfo(args.changed_from) + except (subprocess.CalledProcessError, OSError, ValueError) as error: + parser.error(f"Cannot read previous release metadata: {error}") # Filter versions if needed versions_to_build = [] for version, buildinfo in sorted(builddata.items(), key=lambda item: item[0], reverse=True): + if previous is not None and not release_changed(version, buildinfo, previous): + continue if args.only_stable_latest or (build_rootless and not build_regular): # For rootless-only builds, default to stable/latest only if "stable" in buildinfo["tags"] or "latest" in buildinfo["tags"]: @@ -130,6 +159,13 @@ def main(): else: versions_to_build.append((version, buildinfo)) + if not versions_to_build: + print("No changed releases to build.") + return + + if args.push_tags: + login() + # Build regular images if build_regular: print("Building regular images...") @@ -142,16 +178,7 @@ def main(): # Build rootless images if build_rootless: print("Building rootless images...") - # For rootless, only build stable and latest unless building both - rootless_versions = [] - if not build_regular or args.only_stable_latest: - for version, buildinfo in builddata.items(): - if "stable" in buildinfo["tags"] or "latest" in buildinfo["tags"]: - rootless_versions.append((version, buildinfo)) - else: - rootless_versions = versions_to_build - - for version, buildinfo in rootless_versions: + for version, buildinfo in versions_to_build: sha256 = buildinfo["sha256"] original_tags = buildinfo["tags"] rootless_tags = generate_rootless_tags(original_tags) diff --git a/tests/test_changed_builds.py b/tests/test_changed_builds.py new file mode 100644 index 00000000..20ca4a57 --- /dev/null +++ b/tests/test_changed_builds.py @@ -0,0 +1,144 @@ +"""Incremental release selection and the actual publishing workflow shell logic.""" + +import copy +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +BASE = { + "2.0.77": {"sha256": "stable-sum", "tags": ["stable", "stable-2.0.77", "2.0.77"]}, + "2.1.23": {"sha256": "experimental-sum", "sha256_arm64": "arm-sum", "tags": ["latest", "2.1.23"]}, +} + + +def build_module(): + spec = importlib.util.spec_from_file_location("build_changed", ROOT / "build.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class ChangedBuildTests(unittest.TestCase): + def setUp(self): + self.build = build_module() + + def test_unchanged_and_removed_aliases_do_not_rebuild(self): + info = copy.deepcopy(BASE["2.1.23"]) + self.assertFalse(self.build.release_changed("2.1.23", info, BASE)) + info["tags"].remove("latest") + self.assertFalse(self.build.release_changed("2.1.23", info, BASE)) + + def test_new_versions_archives_and_added_tags_rebuild(self): + info = copy.deepcopy(BASE["2.1.23"]) + self.assertTrue(self.build.release_changed("2.1.24", info, BASE)) + for key in ("sha256", "sha256_arm64"): + changed = {**info, key: "new-sum"} + self.assertTrue(self.build.release_changed("2.1.23", changed, BASE)) + del info["sha256_arm64"] + self.assertTrue(self.build.release_changed("2.1.23", info, BASE)) + info = copy.deepcopy(BASE["2.1.23"]) + info["tags"].append("stable") + self.assertTrue(self.build.release_changed("2.1.23", info, BASE)) + + def test_experimental_update_does_not_rebuild_stable_in_either_variant(self): + data = copy.deepcopy(BASE) + data["2.1.24"] = data.pop("2.1.23") + data["2.1.24"]["tags"] = ["latest", "2.1.24"] + for flags, count in (([], 1), (["--rootless"], 1), (["--both"], 2), (["--both", "--only-stable-latest"], 2)): + with self.subTest(flags=flags), patch.object(self.build.json, "load", return_value=data), \ + patch.object(self.build, "load_previous_buildinfo", return_value=BASE), \ + patch.object(self.build, "build_and_push") as build, \ + patch("sys.argv", ["build.py", "--changed-from", "previous", *flags]): + self.build.main() + self.assertEqual(build.call_count, count) + self.assertEqual({call.args[1] for call in build.call_args_list}, {"2.1.24"}) + + def test_no_changes_do_not_login_or_build(self): + with patch.object(self.build.json, "load", return_value=BASE), \ + patch.object(self.build, "load_previous_buildinfo", return_value=BASE), \ + patch.object(self.build, "login") as login, \ + patch.object(self.build, "build_and_push") as build, \ + patch("sys.argv", ["build.py", "--changed-from", "previous", "--both", "--push-tags"]): + self.build.main() + login.assert_not_called() + build.assert_not_called() + + def test_invalid_baseline_fails_before_building(self): + with patch.object(self.build.json, "load", return_value=BASE), \ + patch.object(self.build, "load_previous_buildinfo", side_effect=ValueError("invalid baseline")), \ + patch.object(self.build, "build_and_push") as build, \ + patch("sys.argv", ["build.py", "--changed-from", "bad"]), \ + self.assertRaises(SystemExit) as error: + self.build.main() + self.assertEqual(error.exception.code, 2) + build.assert_not_called() + + +class BuildWorkflowTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.work = Path(self.temp.name) + (self.work / "docker").mkdir() + (self.work / "docker/Dockerfile").write_text("FROM scratch\n") + (self.work / "buildinfo.json").write_text(json.dumps(BASE)) + build = self.work / "build.py" + build.write_text('#!/usr/bin/env python3\nimport json,sys\nfrom pathlib import Path\nPath("build-args.json").write_text(json.dumps(sys.argv[1:]))\n') + build.chmod(0o755) + self.git("init", "--quiet") + self.commit() + self.before = self.git("rev-parse", "HEAD").stdout.strip() + + def git(self, *args): + return subprocess.run(["git", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", *args], + cwd=self.work, check=True, capture_output=True, text=True) + + def commit(self): + self.git("add", ".") + self.git("commit", "--quiet", "-m", "fixture") + + def run_workflow(self, previous): + workflow = (ROOT / ".github/workflows/docker-build.yml").read_text() + script = textwrap.dedent(workflow.rsplit(" run: |\n", 1)[1]) + result = subprocess.run(["bash", "-e", "-c", script], cwd=self.work, + env={**os.environ, "PREVIOUS_SHA": previous}, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads((self.work / "build-args.json").read_text()) + + def test_metadata_only_update_uses_baseline(self): + (self.work / "buildinfo.json").write_text("{}") + self.commit() + args = self.run_workflow(self.before) + self.assertEqual(args[-2:], ["--changed-from", self.before]) + module = build_module() + module.__file__ = str(self.work / "build.py") + self.assertEqual(module.load_previous_buildinfo(self.before), BASE) + + def test_runtime_change_forces_full_rebuild(self): + (self.work / "docker/Dockerfile").write_text("FROM scratch\n# runtime changed\n") + self.commit() + self.assertNotIn("--changed-from", self.run_workflow(self.before)) + + def test_manual_or_missing_baseline_rebuilds_all(self): + for previous in ("", "0" * 40): + self.assertNotIn("--changed-from", self.run_workflow(previous)) + + def test_scheduled_update_passes_before_and_after_commits(self): + update = (ROOT / ".github/workflows/update.yml").read_text() + self.assertIn("previous_sha: ${{ needs.check.outputs.before_sha }}", update) + self.assertIn("checkout_ref: ${{ needs.check.outputs.head_sha }}", update) + self.assertIn('echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"', update) + workflow = (ROOT / ".github/workflows/docker-build.yml").read_text() + self.assertIn("ref: ${{ inputs.checkout_ref || github.sha }}", workflow) + self.assertIn("fetch-depth: 0", workflow) + + +if __name__ == "__main__": + unittest.main()