From ff2074b5f94e4eee83c1687ef8f3c0921017bf05 Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Thu, 12 Nov 2026 17:22:38 -0700 Subject: [PATCH 1/6] Build pull requests in the build branches' stages The PR build of each distribution and platform is no longer one job that needs re-running after six hours: a plan job lists vinca-gha's stages (--json), pr-stages.yaml runs them in fixed stage slots, one matrix job per batch of recipes, and the packages pass between the stages as artifacts, read as a local channel. A collect job saves them as the PR's build cache and runs the ABI check. pr-stages.yaml is generated by tools/pr_stages.py (rs check checks it). vinca is pinned to the --json change on top of the previous pin. --- .github/workflows/pr-stage-job.yaml | 169 ++++ .github/workflows/pr-stages.yaml | 1225 +++++++++++++++++++++++++++ .github/workflows/testpr.yaml | 215 +---- pixi.lock | 12 +- pixi.toml | 2 +- tools/maintenance.py | 4 + tools/pr_stages.py | 359 ++++++++ tools/robostack.py | 3 + tools/tests/test_ci_build.py | 4 +- tools/tests/test_pr_stages.py | 78 ++ 10 files changed, 1854 insertions(+), 217 deletions(-) create mode 100644 .github/workflows/pr-stage-job.yaml create mode 100644 .github/workflows/pr-stages.yaml create mode 100644 tools/pr_stages.py create mode 100644 tools/tests/test_pr_stages.py diff --git a/.github/workflows/pr-stage-job.yaml b/.github/workflows/pr-stage-job.yaml new file mode 100644 index 000000000..f32ece0a3 --- /dev/null +++ b/.github/workflows/pr-stage-job.yaml @@ -0,0 +1,169 @@ +# One build job of a staged pull-request build (pr-stages.yaml): builds its recipes with +# the packages of the earlier stages (and the restored cache) as a local channel, and +# passes what it built on to the later stages and the cache as an artifact. +name: PR stage job + +on: + workflow_call: + inputs: + distro: + required: true + type: string + platform: + required: true + type: string + os: + required: true + type: string + recipes: + description: Space-separated recipes + required: true + type: string + artifact: + description: Name of the artifact with the packages the job builds + required: true + type: string + +env: + PYTHONIOENCODING: utf-8 + DISTRO: ${{ inputs.distro }} + PLATFORM: ${{ inputs.platform }} + RECIPES: ${{ inputs.recipes }} + ARTIFACT: ${{ inputs.artifact }} + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} + # rattler_build() in tools/robostack.py adds it as a channel (relative to the repository) + ROBOSTACK_LOCAL_CHANNEL: pr-channel + # rattler-build writes to distros//work/output, or to CONDA_BLD_PATH (win-64) + OUTPUT_DIR: ${{ inputs.platform == 'win-64' && 'D:/bld' || format('distros/{0}/work/output', inputs.distro) }} + +jobs: + build: + runs-on: ${{ inputs.os }} + timeout-minutes: 350 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 && git config --global core.longpaths true + # pixi's package cache on D:, with the builds' (see the Windows step below) + - name: "pixi's cache on D:" + if: runner.os == 'Windows' + shell: pwsh + run: '"PIXI_CACHE_DIR=D:\pixi-cache" | Out-File -FilePath $env:GITHUB_ENV -Append' + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + + - name: Windows workarounds (short build path, long paths, PATH and SDK clean-up) + if: runner.os == 'Windows' + shell: pwsh + run: | + # Defender's real-time scan slows down installing and deleting the environments' + # files several times over; the runner is thrown away after the job + Set-MpPreference -DisableRealtimeMonitoring $true + # the builds, rattler's package cache (on the same drive, for hardlinks) and the + # temporary files on D:, about three times as fast as C: on GitHub's runners + "CONDA_BLD_PATH=D:\bld\", "RATTLER_CACHE_DIR=D:\rattler", "TMP=D:\tmp", "TEMP=D:\tmp" | + Out-File -FilePath $env:GITHUB_ENV -Append + New-Item -ItemType Directory -Force -Path D:\bld, D:\rattler, D:\tmp | Out-Null + New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force + Remove-Item -Recurse -Force C:\Strawberry -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0" -ErrorAction SilentlyContinue + + # Spotlight indexes every file the environments and builds write, using up to a + # core or two of the runner's three or four; the runner is thrown away after the job + - name: Turn off Spotlight + if: runner.os == 'macOS' + run: sudo mdutil -a -i off || true + + - name: Clean up PATH + if: runner.os == 'Windows' + uses: egor-tensin/cleanup-path@v5 + with: + dirs: 'C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin' + + - name: Disable git auto-maintenance in source caches + shell: bash -l {0} + run: git config --global maintenance.auto false + + - name: Prepare the work directory + shell: bash -l {0} + run: pixi run rs "$DISTRO" prepare + # the plan job's recipes (generated once for all the jobs) + - uses: actions/download-artifact@v7 + with: + name: recipes-${{ inputs.distro }}-${{ inputs.platform }} + path: distros/${{ inputs.distro }}/work/recipes + # everything the earlier stages built, and the restored cache + - uses: actions/download-artifact@v7 + with: + pattern: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-* + merge-multiple: true + path: pr-channel + - name: Index the earlier stages' packages + shell: bash -l {0} + run: | + mkdir -p "pr-channel/$PLATFORM" pr-channel/noarch + shopt -s nullglob + packages=(pr-channel/*/*.conda) + echo "${#packages[@]} packages from the earlier stages and the cache" + pixi run rattler-index fs pr-channel --force + + - name: Build recipes + id: build-recipes + shell: bash -l {0} + run: | + set +e + args=() + for recipe in $RECIPES; do args+=(--recipe "./recipes/$recipe"); done + # shorter work dirs for MAX_PATH on Windows + [ "$PLATFORM" == "win-64" ] && args+=(--no-build-id) + # the cache's packages are in pr-channel; build-ci builds independent recipes side by side + pixi run rs "$DISTRO" build-ci "${args[@]}" --skip-existing=all 2>&1 | tee rattler-build.log + exit "${PIPESTATUS[0]}" + + - name: Summarize build result + if: always() + shell: bash -l {0} + run: | + pixi run python tools/summarize_rattler_build.py --log rattler-build.log \ + --platform "$DISTRO $PLATFORM" --outcome "${{ steps.build-recipes.outcome }}" >> "$GITHUB_STEP_SUMMARY" + + # also after a failure: what was built goes into the cache + - name: Collect the packages + if: ${{ success() || failure() }} + shell: bash -l {0} + run: | + shopt -s nullglob + mkdir -p pkgs + for subdir in "$PLATFORM" noarch; do + files=("$OUTPUT_DIR/$subdir"/*.conda) + if (( ${#files[@]} )); then mkdir -p "pkgs/$subdir" && cp "${files[@]}" "pkgs/$subdir/"; fi + done + # keeps the / directories in the artifact (it starts at the files' common directory) + (cd pkgs && printf '%s\n' ./*/*.conda) > "pkgs/manifest-$ARTIFACT.txt" + cat "pkgs/manifest-$ARTIFACT.txt" + - name: Upload the packages + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v6 + with: + name: ${{ inputs.artifact }} + path: pkgs + compression-level: 0 # .conda files are compressed + overwrite: true # re-runs of a failed job + retention-days: 3 + + - name: Upload build log + if: always() + uses: actions/upload-artifact@v6 + with: + name: build-log-${{ inputs.artifact }}-${{ github.run_attempt }} + path: rattler-build.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/pr-stages.yaml b/.github/workflows/pr-stages.yaml new file mode 100644 index 000000000..06035be2e --- /dev/null +++ b/.github/workflows/pr-stages.yaml @@ -0,0 +1,1225 @@ +# Generated by tools/pr_stages.py (python tools/pr_stages.py workflow); `pixi run rs check` +# checks that it is up to date. +# +# One distribution's pull-request build on one platform, in the build branches' stages: +# the plan job generates the recipes and lists each stage's jobs (vinca-gha --json), +# each stage_ slot builds its stage's jobs side by side (pr-stage-job.yaml), and +# collect keeps the packages as the pull request's build cache. Packages pass from +# stage to stage as artifacts (pkgs---*), which the later jobs read as +# a local channel: no secrets, so pull requests from forks build the same way. +name: PR stages + +on: + workflow_call: + inputs: + distro: + required: true + type: string + platform: + required: true + type: string + os: + description: The platform's runner + required: true + type: string + +env: + PYTHONIOENCODING: utf-8 + DISTRO: ${{ inputs.distro }} + PLATFORM: ${{ inputs.platform }} + # while uploads go to a test channel, packages already there aren't built again + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} + # the pull request's packages, /*.conda; restored and saved on Linux for every platform + CACHE_DIR: pr-cache + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + count: ${{ steps.plan.outputs.count }} + stages: ${{ steps.plan.outputs.stages }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - name: Generate recipes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi + echo "Recipe generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done + - name: Upload the recipes + uses: actions/upload-artifact@v6 + with: + name: recipes-${{ inputs.distro }}-${{ inputs.platform }} + path: distros/${{ inputs.distro }}/work/recipes + if-no-files-found: ignore + overwrite: true + retention-days: 7 + - name: Restore build cache + uses: actions/cache/restore@v5 + with: + path: ${{ env.CACHE_DIR }} + key: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }} + restore-keys: | + pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}- + # Temporary rebuild controls live in distros//ci.yaml. + - name: Apply ci.yaml cache controls + run: | + for subdir in "$PLATFORM" noarch; do + mkdir -p "$CACHE_DIR/$subdir" + pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR/$subdir" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" + done + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages restored from the cache" + for package in "${packages[@]}"; do basename "$package"; done + # the stage jobs' download pattern always matches this artifact + echo "cache" > "$CACHE_DIR/manifest-cache.txt" + - name: Upload the cached packages + uses: actions/upload-artifact@v6 + with: + name: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-cache + path: ${{ env.CACHE_DIR }} + compression-level: 0 # .conda files are compressed + overwrite: true + retention-days: 3 + - name: Estimate the build time + run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR/$PLATFORM" --platform "$PLATFORM" + - name: Plan the stages + id: plan + run: pixi run python tools/pr_stages.py plan --distro "$DISTRO" --platform "$PLATFORM" --cache-dir "$CACHE_DIR" + + stage_0: + needs: plan + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 0 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[0] }} + name: 0 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_1: + needs: [plan, stage_0] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 1 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[1] }} + name: 1 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_2: + needs: [plan, stage_1] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 2 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[2] }} + name: 2 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_3: + needs: [plan, stage_2] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 3 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[3] }} + name: 3 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_4: + needs: [plan, stage_3] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 4 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[4] }} + name: 4 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_5: + needs: [plan, stage_4] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 5 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[5] }} + name: 5 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_6: + needs: [plan, stage_5] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 6 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[6] }} + name: 6 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_7: + needs: [plan, stage_6] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 7 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[7] }} + name: 7 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_8: + needs: [plan, stage_7] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 8 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[8] }} + name: 8 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_9: + needs: [plan, stage_8] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 9 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[9] }} + name: 9 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_10: + needs: [plan, stage_9] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 10 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[10] }} + name: 10 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_11: + needs: [plan, stage_10] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 11 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[11] }} + name: 11 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_12: + needs: [plan, stage_11] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 12 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[12] }} + name: 12 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_13: + needs: [plan, stage_12] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 13 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[13] }} + name: 13 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_14: + needs: [plan, stage_13] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 14 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[14] }} + name: 14 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_15: + needs: [plan, stage_14] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 15 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[15] }} + name: 15 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_16: + needs: [plan, stage_15] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 16 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[16] }} + name: 16 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_17: + needs: [plan, stage_16] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 17 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[17] }} + name: 17 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_18: + needs: [plan, stage_17] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 18 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[18] }} + name: 18 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_19: + needs: [plan, stage_18] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 19 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[19] }} + name: 19 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_20: + needs: [plan, stage_19] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 20 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[20] }} + name: 20 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_21: + needs: [plan, stage_20] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 21 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[21] }} + name: 21 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_22: + needs: [plan, stage_21] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 22 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[22] }} + name: 22 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_23: + needs: [plan, stage_22] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 23 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[23] }} + name: 23 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_24: + needs: [plan, stage_23] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 24 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[24] }} + name: 24 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_25: + needs: [plan, stage_24] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 25 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[25] }} + name: 25 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_26: + needs: [plan, stage_25] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 26 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[26] }} + name: 26 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_27: + needs: [plan, stage_26] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 27 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[27] }} + name: 27 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_28: + needs: [plan, stage_27] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 28 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[28] }} + name: 28 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_29: + needs: [plan, stage_28] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 29 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[29] }} + name: 29 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_30: + needs: [plan, stage_29] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 30 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[30] }} + name: 30 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_31: + needs: [plan, stage_30] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 31 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[31] }} + name: 31 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_32: + needs: [plan, stage_31] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 32 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[32] }} + name: 32 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_33: + needs: [plan, stage_32] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 33 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[33] }} + name: 33 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_34: + needs: [plan, stage_33] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 34 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[34] }} + name: 34 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_35: + needs: [plan, stage_34] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 35 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[35] }} + name: 35 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_36: + needs: [plan, stage_35] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 36 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[36] }} + name: 36 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_37: + needs: [plan, stage_36] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 37 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[37] }} + name: 37 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_38: + needs: [plan, stage_37] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 38 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[38] }} + name: 38 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_39: + needs: [plan, stage_38] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 39 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[39] }} + name: 39 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_40: + needs: [plan, stage_39] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 40 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[40] }} + name: 40 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_41: + needs: [plan, stage_40] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 41 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[41] }} + name: 41 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_42: + needs: [plan, stage_41] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 42 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[42] }} + name: 42 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_43: + needs: [plan, stage_42] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 43 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[43] }} + name: 43 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_44: + needs: [plan, stage_43] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 44 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[44] }} + name: 44 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_45: + needs: [plan, stage_44] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 45 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[45] }} + name: 45 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_46: + needs: [plan, stage_45] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 46 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[46] }} + name: 46 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_47: + needs: [plan, stage_46] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 47 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[47] }} + name: 47 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_48: + needs: [plan, stage_47] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 48 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[48] }} + name: 48 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_49: + needs: [plan, stage_48] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 49 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[49] }} + name: 49 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_50: + needs: [plan, stage_49] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 50 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[50] }} + name: 50 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_51: + needs: [plan, stage_50] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 51 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[51] }} + name: 51 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_52: + needs: [plan, stage_51] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 52 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[52] }} + name: 52 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_53: + needs: [plan, stage_52] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 53 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[53] }} + name: 53 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_54: + needs: [plan, stage_53] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 54 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[54] }} + name: 54 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_55: + needs: [plan, stage_54] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 55 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[55] }} + name: 55 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_56: + needs: [plan, stage_55] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 56 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[56] }} + name: 56 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_57: + needs: [plan, stage_56] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 57 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[57] }} + name: 57 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_58: + needs: [plan, stage_57] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 58 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[58] }} + name: 58 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_59: + needs: [plan, stage_58] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 59 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[59] }} + name: 59 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_60: + needs: [plan, stage_59] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 60 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[60] }} + name: 60 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_61: + needs: [plan, stage_60] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 61 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[61] }} + name: 61 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_62: + needs: [plan, stage_61] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 62 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[62] }} + name: 62 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + stage_63: + needs: [plan, stage_62] + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 63 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[63] }} + name: 63 ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + + # The packages of every stage as the pull request's build cache (a new push builds + # only what changed), then the ABI check of what the pull request built. + collect: + needs: [plan, stage_0, stage_1, stage_2, stage_3, stage_4, stage_5, stage_6, stage_7, stage_8, stage_9, stage_10, stage_11, stage_12, stage_13, stage_14, stage_15, stage_16, stage_17, stage_18, stage_19, stage_20, stage_21, stage_22, stage_23, stage_24, stage_25, stage_26, stage_27, stage_28, stage_29, stage_30, stage_31, stage_32, stage_33, stage_34, stage_35, stage_36, stage_37, stage_38, stage_39, stage_40, stage_41, stage_42, stage_43, stage_44, stage_45, stage_46, stage_47, stage_48, stage_49, stage_50, stage_51, stage_52, stage_53, stage_54, stage_55, stage_56, stage_57, stage_58, stage_59, stage_60, stage_61, stage_62, stage_63] + if: ${{ always() && needs.plan.result == 'success' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - uses: actions/download-artifact@v7 + with: + pattern: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-* + merge-multiple: true + path: ${{ env.CACHE_DIR }} + - name: See the packages + run: | + rm -f "$CACHE_DIR"/manifest-*.txt + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages" + for package in "${packages[@]}"; do basename "$package"; done + - name: Save build cache + id: save-cache + uses: actions/cache/save@v5 + with: + path: ${{ env.CACHE_DIR }} + key: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + + # Only the newest cache of a pull request/distribution/platform is restored; the ones + # of earlier runs and attempts only push others out of the repository's 10 GB, so + # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. + - name: Drop this pull request's superseded build caches + if: ${{ steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PREFIX: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}- + KEEP: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + run: | + # cache/save reports success even on upload failure: verify the replacement exists. + if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \ + --jq "if any(.[]; .key == \"$KEEP\") then .[] | select(.key != \"$KEEP\") | .id else error(\"New cache not found\") end"); then + echo "::warning::Could not verify the new build cache; keeping all previous caches." + exit 0 + fi + while read -r id; do + [ -n "$id" ] || continue + pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" + done <<< "$ids" + + # Compare what the pull request built with the released builds, so single packages + # can be bumped without a full rebuild; it reports and never fails the build. + - name: ABI check against the released builds + id: abi-check + if: inputs.platform == 'linux-64' + continue-on-error: true + run: | + pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR/$PLATFORM" --summary "abi-check-$DISTRO.md" + cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" + - name: Upload the ABI check + if: ${{ steps.abi-check.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: abi-check-${{ inputs.distro }} + path: abi-check-${{ inputs.distro }}.md + retention-days: 7 + + # without recipes to build there is no recipes artifact, and nothing to stage + - name: Prepare the work directory + if: needs.plan.outputs.count != '0' + run: pixi run rs "$DISTRO" prepare + - if: needs.plan.outputs.count != '0' + uses: actions/download-artifact@v7 + with: + name: recipes-${{ inputs.distro }}-${{ inputs.platform }} + path: distros/${{ inputs.distro }}/work/recipes + - name: Generate the staged build workflows (catches post-merge problems) + if: needs.plan.outputs.count != '0' + run: | + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi + echo "Workflow generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index c93483fdf..ea289d632 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -136,6 +136,8 @@ jobs: pixi run rs "$DISTRO" check-orphaned-patches pixi run rs "$DISTRO" check-patches + # Each distribution and platform in the build branches' stages, one job per batch of + # recipes (pr-stages.yaml, generated by tools/pr_stages.py). build: permissions: actions: write # drop this pull request's superseded build caches @@ -148,214 +150,11 @@ jobs: distro: ${{ fromJSON(needs.plan.outputs.distros) }} target: ${{ fromJSON(needs.plan.outputs.targets) }} # tools/robostack.py BUILD_TARGETS name: ${{ matrix.distro }} ${{ matrix.target.platform }} - runs-on: ${{ matrix.target.os }} - env: - DISTRO: ${{ matrix.distro }} - PLATFORM: ${{ matrix.target.platform }} - # while uploads go to a test channel, packages already there aren't built again - ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} - # rattler-build writes to distros//work/output, or to CONDA_BLD_PATH (win-64) - CACHE_DIR: ${{ matrix.target.platform == 'win-64' && 'D:/bld/win-64' || format('distros/{0}/work/output/{1}', matrix.distro, matrix.target.platform) }} - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - fetch-depth: 0 - - # git auto-maintenance can leave a lock file in the git clones pixi caches - # (e.g. vinca's), which breaks every later run restoring that cache - - name: Disable git auto-maintenance - run: git config --global maintenance.auto false && git config --global gc.auto 0 && git config --global core.longpaths true - # pixi's package cache on D:, with the builds' (see the Windows step below) - - name: "pixi's cache on D:" - if: runner.os == 'Windows' - shell: pwsh - run: '"PIXI_CACHE_DIR=D:\pixi-cache" | Out-File -FilePath $env:GITHUB_ENV -Append' - - uses: prefix-dev/setup-pixi@v0.11.0 - with: - locked: true # the lock file must match pixi.toml - pixi-version: v0.81.0 - cache-write: false # saved from main only (pixi-cache.yaml) - - - name: Windows workarounds (short build path, long paths, PATH and SDK clean-up) - if: runner.os == 'Windows' - shell: pwsh - run: | - # Defender's real-time scan slows down installing and deleting the environments' - # files several times over; the runner is thrown away after the job - Set-MpPreference -DisableRealtimeMonitoring $true - # the builds, rattler's package cache (on the same drive, for hardlinks) and the - # temporary files on D:, about three times as fast as C: on GitHub's runners - "CONDA_BLD_PATH=D:\bld\", "RATTLER_CACHE_DIR=D:\rattler", "TMP=D:\tmp", "TEMP=D:\tmp" | - Out-File -FilePath $env:GITHUB_ENV -Append - New-Item -ItemType Directory -Force -Path D:\bld, D:\rattler, D:\tmp | Out-Null - Get-PSDrive C, D | Format-Table Name, @{n="free GB"; e={[math]::Round($_.Free / 1GB)}} - New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force - Remove-Item -Recurse -Force C:\Strawberry -ErrorAction SilentlyContinue - Remove-Item -Recurse -Force "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0" -ErrorAction SilentlyContinue - - # Spotlight indexes every file the environments and builds write, using up to a - # core or two of the runner's three or four; the runner is thrown away after the job - - name: Turn off Spotlight - if: runner.os == 'macOS' - run: sudo mdutil -a -i off || true - - - name: Clean up PATH - if: runner.os == 'Windows' - uses: egor-tensin/cleanup-path@v5 - with: - dirs: 'C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin' - - - name: Disable git auto-maintenance in source caches - shell: bash -l {0} - run: git config --global maintenance.auto false - - - name: Generate recipes - shell: bash -l {0} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - for attempt in 1 2 3; do - if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi - if [[ "${attempt}" == "3" ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi - echo "Recipe generation attempt ${attempt} failed; retrying..." >&2 - sleep 15 - done - - - name: Restore build cache - uses: actions/cache/restore@v5 - with: - path: ${{ env.CACHE_DIR }} - key: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }} - restore-keys: | - ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}- - - # Temporary rebuild controls live in distros//ci.yaml. - - name: Apply ci.yaml cache controls - shell: bash -l {0} - run: | - pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" - mkdir -p "$CACHE_DIR" - pixi run rattler-index fs "$CACHE_DIR/.." --force - - - name: See packages restored by cache - shell: bash -l {0} - run: | - shopt -s nullglob - packages=("$CACHE_DIR"/*.conda) - echo "${#packages[@]} packages restored from the cache" - for package in "${packages[@]}"; do basename "$package"; done - - - name: Estimate the build time - shell: bash -l {0} - run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR" --platform "$PLATFORM" - - - name: Build recipes - id: build-recipes - # Leave time for cleanup and cache upload before the six-hour job limit. - timeout-minutes: 340 - shell: bash -l {0} - run: | - set +e - extra=() - # shorter work dirs for MAX_PATH on Windows - [ "$PLATFORM" == "win-64" ] && extra+=(--no-build-id) - # independent recipes side by side, with a shared jobserver keeping them at the - # runner's cores in total; Windows has no jobserver, and was no faster with two - [ "$PLATFORM" != "win-64" ] && extra+=(--parallel-builds "3:$(getconf _NPROCESSORS_ONLN)") - build=(python tools/robostack.py "$DISTRO" build --target-platform "$PLATFORM" "${extra[@]}") - if [ "$PLATFORM" != "win-64" ]; then - build=(python tools/ci_build.py --cache-dir "$CACHE_DIR" --timeout 19800 -- "${build[@]}") - fi - pixi run "${build[@]}" 2>&1 | tee rattler-build.log - status=${PIPESTATUS[0]} - echo "exit-code=${status}" >> "$GITHUB_OUTPUT" - exit "$status" - - - name: Summarize build result - if: always() - shell: bash -l {0} - run: | - pixi run python tools/summarize_rattler_build.py --log rattler-build.log \ - --platform "$DISTRO $PLATFORM" --outcome "${{ steps.build-recipes.outcome }}" >> "$GITHUB_STEP_SUMMARY" - - # Compare what the pull request built with the released builds, so single packages - # can be bumped without a full rebuild; it reports and never fails the build. - - name: ABI check against the released builds - id: abi-check - if: ${{ always() && matrix.target.platform == 'linux-64' && contains(fromJSON('["success","failure"]'), steps.build-recipes.outcome) }} - continue-on-error: true - shell: bash -l {0} - run: | - pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR" --summary "abi-check-$DISTRO.md" - cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" - - - name: Upload the ABI check - if: ${{ always() && steps.abi-check.outcome == 'success' }} - uses: actions/upload-artifact@v6 - with: - name: abi-check-${{ matrix.distro }} - path: abi-check-${{ matrix.distro }}.md - retention-days: 7 - - - name: Upload build log - if: always() - uses: actions/upload-artifact@v6 - with: - name: build-log-${{ matrix.distro }}-${{ matrix.target.platform }}-${{ github.run_id }}-${{ github.run_attempt }} - path: rattler-build.log - if-no-files-found: warn - retention-days: 7 - - - name: Drop the Windows package a cancellation may have cut short - # Windows builds one package at a time; POSIX builds are cleaned by ci_build.py. - if: ${{ always() && runner.os == 'Windows' && steps.build-recipes.outcome == 'cancelled' }} - shell: bash -l {0} - run: | - newest=$(ls -t "$CACHE_DIR"/*.conda 2>/dev/null | head -1) - if [ -n "$newest" ]; then echo "Removing $newest"; rm -f "$newest"; fi - - - name: Save build cache - id: save-cache - if: ${{ always() && contains(fromJSON('["success","failure","cancelled"]'), steps.build-recipes.outcome) && (runner.os == 'Windows' || steps.build-recipes.outputs.cache-ready == 'true') }} - uses: actions/cache/save@v5 - with: - path: ${{ env.CACHE_DIR }} - key: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} - - # Only the newest cache of a pull request/distribution/platform is restored; the ones - # of earlier runs and attempts only push others out of the repository's 10 GB, so - # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. - - name: Drop this pull request's superseded build caches - if: ${{ always() && steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} - continue-on-error: true - shell: bash -l {0} - env: - GH_TOKEN: ${{ github.token }} - PREFIX: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}- - KEEP: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} - run: | - # cache/save reports success even on upload failure: verify the replacement exists. - if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \ - --jq "if any(.[]; .key == \"$KEEP\") then .[] | select(.key != \"$KEEP\") | .id else error(\"New cache not found\") end"); then - echo "::warning::Could not verify the new build cache; keeping all previous caches." - exit 0 - fi - while read -r id; do - [ -n "$id" ] || continue - pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" - done <<< "$ids" - - - name: Generate the staged build workflows (catches post-merge problems) - shell: bash -l {0} - run: | - for attempt in 1 2 3; do - if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi - if [[ "${attempt}" == "3" ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi - echo "Workflow generation attempt ${attempt} failed; retrying..." >&2 - sleep 15 - done + uses: ./.github/workflows/pr-stages.yaml + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.target.platform }} + os: ${{ matrix.target.os }} # The ABI checks of all distributions as one pull-request comment, updated on every # push. Pull requests from forks get a read-only token, so the job is skipped for them: diff --git a/pixi.lock b/pixi.lock index ab0edb4f3..941590414 100644 --- a/pixi.lock +++ b/pixi.lock @@ -90,7 +90,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -170,7 +170,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -243,7 +243,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/tk-8.6.13-ha6c374e_4.conda - conda: https://prefix.dev/conda-forge/osx-64/yaml-0.2.5-had63097_3.conda - conda: https://prefix.dev/conda-forge/osx-64/zstd-1.5.7-hbc1a06c_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -316,7 +316,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - conda: https://prefix.dev/conda-forge/osx-arm64/yaml-0.2.5-h74c22ad_3.conda - conda: https://prefix.dev/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -389,7 +389,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/vcomp14-14.51.36247-habf1de7_41.conda - conda: https://prefix.dev/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://prefix.dev/conda-forge/win-64/zstd-1.5.7-h534d264_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -3289,7 +3289,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 size: 387535 timestamp: 1786599623274 -- pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 +- pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 name: vinca version: 0.2.0 requires_dist: diff --git a/pixi.toml b/pixi.toml index c76cfff92..8cafeac9e 100644 --- a/pixi.toml +++ b/pixi.toml @@ -36,7 +36,7 @@ git = "*" [pypi-dependencies] # One vinca for all distributions. -vinca = { git = "https://github.com/RoboStack/vinca.git", rev = "74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5" } +vinca = { git = "https://github.com/EzraBrooks/vinca.git", rev = "b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854" } # For local development, temporarily use (and regenerate pixi.lock): # vinca = { path = "../vinca", editable = true } diff --git a/tools/maintenance.py b/tools/maintenance.py index 8c0f81d49..1a2ee5519 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -1403,6 +1403,10 @@ def check() -> Result: text = wf.read_text() if "\non:" not in "\n" + text or "\ntrue:" in "\n" + text: problems.append(f"{wf.name}: no top-level `on:` trigger") + import pr_stages + + if not pr_stages.WORKFLOW.is_file() or pr_stages.WORKFLOW.read_text() != pr_stages.workflow(): + problems.append(f"{pr_stages.WORKFLOW.relative_to(rs.ROOT)} is out of date (python tools/pr_stages.py workflow)") sort = subprocess.run(["pixi", "run", "sort", "--check"], cwd=rs.ROOT, capture_output=True, text=True) if sort.returncode: problems.append("YAML files are not sorted (pixi run sort):\n" + tail(sort.stdout + sort.stderr, 10)) diff --git a/tools/pr_stages.py b/tools/pr_stages.py new file mode 100644 index 000000000..9abb4a6e4 --- /dev/null +++ b/tools/pr_stages.py @@ -0,0 +1,359 @@ +"""The staged pull-request build: the build-branch stages (vinca-gha) as matrix jobs. + + python tools/pr_stages.py plan --distro --platform

--cache-dir

+ python tools/pr_stages.py workflow [--check] + +GitHub can't add jobs to a running workflow, so .github/workflows/pr-stages.yaml has a +fixed number of stage slots (SLOTS), stage_0 ... stage_, each after the one +before. `plan` (its plan job) lists each stage's jobs from `vinca-gha --json`, leaving +out the recipes whose every build is in the restored pull-request cache; a slot takes +its stage's jobs as its matrix and is skipped when there is no such stage. `workflow` +writes pr-stages.yaml (`rs check` checks that it is up to date). +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import build_timings +import robostack as rs + +SLOTS = 64 # lyrical's full linux-64 rebuild has 47 stages at --batch_size 25 +MAX_NAME = 60 # job names are a stage's packages, which can be many +WORKFLOW = rs.ROOT / ".github" / "workflows" / "pr-stages.yaml" + + +def cached_recipes(recipes: Path, platform: str, cache_dir: Path) -> set[str]: + """The recipes whose every build (one per variant) is in the cache (/*.conda).""" + names = {p.parent.name for p in recipes.glob("*/recipe.yaml")} + variants = build_timings.variant_builds(recipes, platform) if names else {} + cached = Counter(f.name.rsplit("-", 2)[0] for f in cache_dir.glob("*/*.conda")) + return {r for r in names if cached[r] >= variants.get(r, 1)} + + +def fit(stages: list[list[dict]], slots: int = SLOTS) -> list[list[dict]]: + """The stages as at most `slots` stages: the ones past the last slot become one job + there (build-ci orders a job's recipes by their dependencies). Each job gets an id + for its artifact, stage and job number.""" + if len(stages) > slots: + rest = [job for stage in stages[slots - 1:] for job in stage] + tail = {"name": f"{len(rest)} jobs of stages {slots - 1}+", "recipes": " ".join(j["recipes"] for j in rest)} + stages = [*stages[:slots - 1], [tail]] + fitted = [] + for k, stage in enumerate(stages): + jobs = [] + for i, job in enumerate(stage): + name = job["name"] if len(job["name"]) <= MAX_NAME else job["name"][:MAX_NAME - 1] + "…" + jobs.append({"id": f"s{k}-{i}", "name": name, "recipes": job["recipes"]}) + fitted.append(jobs) + return fitted + + +def plan(distro: str, platform: str, cache_dir: Path, batch_size: int) -> list[list[dict]]: + w = rs.work_dir(distro) + recipes = w / "recipes" + cached = cached_recipes(recipes, platform, cache_dir) if recipes.is_dir() else set() + # vinca-gha stages what is in ./recipes; the cached recipes count as built, like the + # ones on the channel that generate-recipes -n leaves out + aside = w / "recipes-cached" + shutil.rmtree(aside, ignore_errors=True) + aside.mkdir() + for r in cached: + (recipes / r).rename(aside / r) + try: + out = w / "pr_stages.json" + cmd = ["vinca-gha", "--platform", platform, "-d", "./recipes", "--batch_size", str(batch_size), + "--json", out.name] + if build_timings.vinca_timings(distro, platform, w / "build_timings.json"): + cmd += ["--timings", "build_timings.json"] + if rs.run(cmd, w): + raise SystemExit("vinca-gha failed") + stages = json.loads(out.read_text())["stages"] + finally: + for r in cached: + (aside / r).rename(recipes / r) + aside.rmdir() + print(f"{len(cached)} recipes fully in the cache; {sum(map(len, stages))} jobs in {len(stages)} stages") + return fit(stages) + + +def workflow() -> str: + """pr-stages.yaml: HEADER, the stage slots, then COLLECT.""" + slots = [] + for k in range(SLOTS): + needs = "plan" if k == 0 else f"[plan, stage_{k - 1}]" + slots.append(SLOT.replace("@K@", str(k)).replace("@NEEDS@", needs)) + every = ", ".join(["plan", *(f"stage_{k}" for k in range(SLOTS))]) + return HEADER + "".join(slots) + COLLECT.replace("@NEEDS@", every) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("plan", help="the stages of one distribution and platform, as GitHub outputs") + p.add_argument("--distro", required=True) + p.add_argument("--platform", required=True) + p.add_argument("--cache-dir", required=True, type=Path) + p.add_argument("--batch-size", type=int, default=25) + w = sub.add_parser("workflow", help=f"write {WORKFLOW.relative_to(rs.ROOT)}") + w.add_argument("--check", action="store_true", help="only check that it is up to date") + args = parser.parse_args() + + if args.command == "workflow": + text = workflow() + if args.check: + if not WORKFLOW.is_file() or WORKFLOW.read_text() != text: + print(f"{WORKFLOW.relative_to(rs.ROOT)} is out of date (python tools/pr_stages.py workflow)") + return 1 + return 0 + WORKFLOW.write_text(text) + return 0 + + stages = plan(args.distro, args.platform, args.cache_dir.resolve(), args.batch_size) + # the slots past the last stage are skipped, but GitHub reads their matrix anyway + padded = stages + [[{"id": "none", "name": "none", "recipes": ""}]] * (SLOTS - len(stages)) + lines = [f"count={len(stages)}", f"stages={json.dumps(padded, separators=(',', ':'))}"] + if output := os.environ.get("GITHUB_OUTPUT"): + with open(output, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + if summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with open(summary, "a", encoding="utf-8") as fh: + jobs = sum(map(len, stages)) + fh.write(f"**{args.distro} {args.platform}**: {jobs} build jobs in {len(stages)} stages\n\n") + for k, stage in enumerate(stages): + print(f"stage {k}: " + "; ".join(job["recipes"] for job in stage)) + return 0 + + +SETUP = """\ + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) +""" + +HEADER = """\ +# Generated by tools/pr_stages.py (python tools/pr_stages.py workflow); `pixi run rs check` +# checks that it is up to date. +# +# One distribution's pull-request build on one platform, in the build branches' stages: +# the plan job generates the recipes and lists each stage's jobs (vinca-gha --json), +# each stage_ slot builds its stage's jobs side by side (pr-stage-job.yaml), and +# collect keeps the packages as the pull request's build cache. Packages pass from +# stage to stage as artifacts (pkgs---*), which the later jobs read as +# a local channel: no secrets, so pull requests from forks build the same way. +name: PR stages + +on: + workflow_call: + inputs: + distro: + required: true + type: string + platform: + required: true + type: string + os: + description: The platform's runner + required: true + type: string + +env: + PYTHONIOENCODING: utf-8 + DISTRO: ${{ inputs.distro }} + PLATFORM: ${{ inputs.platform }} + # while uploads go to a test channel, packages already there aren't built again + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} + # the pull request's packages, /*.conda; restored and saved on Linux for every platform + CACHE_DIR: pr-cache + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + count: ${{ steps.plan.outputs.count }} + stages: ${{ steps.plan.outputs.stages }} + steps: +""" + SETUP + """\ + - name: Generate recipes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi + echo "Recipe generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done + - name: Upload the recipes + uses: actions/upload-artifact@v6 + with: + name: recipes-${{ inputs.distro }}-${{ inputs.platform }} + path: distros/${{ inputs.distro }}/work/recipes + if-no-files-found: ignore + overwrite: true + retention-days: 7 + - name: Restore build cache + uses: actions/cache/restore@v5 + with: + path: ${{ env.CACHE_DIR }} + key: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }} + restore-keys: | + pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}- + # Temporary rebuild controls live in distros//ci.yaml. + - name: Apply ci.yaml cache controls + run: | + for subdir in "$PLATFORM" noarch; do + mkdir -p "$CACHE_DIR/$subdir" + pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR/$subdir" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" + done + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages restored from the cache" + for package in "${packages[@]}"; do basename "$package"; done + # the stage jobs' download pattern always matches this artifact + echo "cache" > "$CACHE_DIR/manifest-cache.txt" + - name: Upload the cached packages + uses: actions/upload-artifact@v6 + with: + name: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-cache + path: ${{ env.CACHE_DIR }} + compression-level: 0 # .conda files are compressed + overwrite: true + retention-days: 3 + - name: Estimate the build time + run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR/$PLATFORM" --platform "$PLATFORM" + - name: Plan the stages + id: plan + run: pixi run python tools/pr_stages.py plan --distro "$DISTRO" --platform "$PLATFORM" --cache-dir "$CACHE_DIR" + +""" + +# a skipped slot doesn't skip the ones after it (!failure() instead of the default +# success()), but a failed one does: its packages are missing for the later stages +SLOT = """\ + stage_@K@: + needs: @NEEDS@ + if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > @K@ }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.stages)[@K@] }} + name: @K@ ${{ matrix.name }} + uses: ./.github/workflows/pr-stage-job.yaml + with: + distro: ${{ inputs.distro }} + platform: ${{ inputs.platform }} + os: ${{ inputs.os }} + recipes: ${{ matrix.recipes }} + artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} + +""" + +COLLECT = """\ + # The packages of every stage as the pull request's build cache (a new push builds + # only what changed), then the ABI check of what the pull request built. + collect: + needs: [@NEEDS@] + if: ${{ always() && needs.plan.result == 'success' }} + runs-on: ubuntu-latest + steps: +""" + SETUP + """\ + - uses: actions/download-artifact@v7 + with: + pattern: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-* + merge-multiple: true + path: ${{ env.CACHE_DIR }} + - name: See the packages + run: | + rm -f "$CACHE_DIR"/manifest-*.txt + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages" + for package in "${packages[@]}"; do basename "$package"; done + - name: Save build cache + id: save-cache + uses: actions/cache/save@v5 + with: + path: ${{ env.CACHE_DIR }} + key: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + + # Only the newest cache of a pull request/distribution/platform is restored; the ones + # of earlier runs and attempts only push others out of the repository's 10 GB, so + # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. + - name: Drop this pull request's superseded build caches + if: ${{ steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PREFIX: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}- + KEEP: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + run: | + # cache/save reports success even on upload failure: verify the replacement exists. + if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \\ + --jq "if any(.[]; .key == \\"$KEEP\\") then .[] | select(.key != \\"$KEEP\\") | .id else error(\\"New cache not found\\") end"); then + echo "::warning::Could not verify the new build cache; keeping all previous caches." + exit 0 + fi + while read -r id; do + [ -n "$id" ] || continue + pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" + done <<< "$ids" + + # Compare what the pull request built with the released builds, so single packages + # can be bumped without a full rebuild; it reports and never fails the build. + - name: ABI check against the released builds + id: abi-check + if: inputs.platform == 'linux-64' + continue-on-error: true + run: | + pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR/$PLATFORM" --summary "abi-check-$DISTRO.md" + cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" + - name: Upload the ABI check + if: ${{ steps.abi-check.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: abi-check-${{ inputs.distro }} + path: abi-check-${{ inputs.distro }}.md + retention-days: 7 + + # without recipes to build there is no recipes artifact, and nothing to stage + - name: Prepare the work directory + if: needs.plan.outputs.count != '0' + run: pixi run rs "$DISTRO" prepare + - if: needs.plan.outputs.count != '0' + uses: actions/download-artifact@v7 + with: + name: recipes-${{ inputs.distro }}-${{ inputs.platform }} + path: distros/${{ inputs.distro }}/work/recipes + - name: Generate the staged build workflows (catches post-merge problems) + if: needs.plan.outputs.count != '0' + run: | + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi + echo "Workflow generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done +""" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/robostack.py b/tools/robostack.py index 96d2f4c67..41cdddbaa 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -386,7 +386,10 @@ def rattler_build(distro: str, args: list[str], skip_existing: bool = True) -> l # With uploads redirected (e.g. to a test channel), later stages of the staged # build need the packages earlier stages uploaded there, and reruns skip them. extra = upload_channel_url() + # the staged pull-request build's packages of earlier stages (tools/pr_stages.py) + local = os.environ.get("ROBOSTACK_LOCAL_CHANNEL", "").strip() cmd = ["rattler-build", "build", "-m", "./conda_build_config.yaml", + *(["-c", (ROOT / local).resolve().as_uri()] if local else []), *(["-c", extra] if extra else []), "-c", channel_url(distro), "-c", CONDA_FORGE, "--channel-priority", "disabled"] return cmd + (["--skip-existing=all"] if skip_existing else []) + args diff --git a/tools/tests/test_ci_build.py b/tools/tests/test_ci_build.py index 3c62f29bc..b06c2af02 100644 --- a/tools/tests/test_ci_build.py +++ b/tools/tests/test_ci_build.py @@ -80,8 +80,8 @@ def test_interruption_stops_descendants_and_preserves_only_finished_archives(tmp @pytest.mark.parametrize("lookup_status,ids,deleted", [(0, "11\n22", "11\n22\n"), (1, "11", ""), (0, "", "")]) def test_cache_pruning_requires_verified_replacement(tmp_path, lookup_status, ids, deleted): - workflow = yaml.safe_load((ROOT / ".github/workflows/testpr.yaml").read_text()) - prune = next(s for s in workflow["jobs"]["build"]["steps"] + workflow = yaml.safe_load((ROOT / ".github/workflows/pr-stages.yaml").read_text()) + prune = next(s for s in workflow["jobs"]["collect"]["steps"] if s.get("name") == "Drop this pull request's superseded build caches") pixi = tmp_path / "pixi" pixi.write_text('''#!/bin/bash diff --git a/tools/tests/test_pr_stages.py b/tools/tests/test_pr_stages.py new file mode 100644 index 000000000..177788b54 --- /dev/null +++ b/tools/tests/test_pr_stages.py @@ -0,0 +1,78 @@ +"""tools/pr_stages.py: the staged pull-request build's plan and stage slots.""" + +import yaml + +import pr_stages +import robostack as rs + + +def job(recipes: str) -> dict: + return {"name": recipes, "recipes": recipes} + + +def test_fit_numbers_the_jobs_for_their_artifacts(): + assert pr_stages.fit([[job("ros2-a")], [job("ros2-b"), job("ros2-c")]]) == [ + [{"id": "s0-0", "name": "ros2-a", "recipes": "ros2-a"}], + [{"id": "s1-0", "name": "ros2-b", "recipes": "ros2-b"}, {"id": "s1-1", "name": "ros2-c", "recipes": "ros2-c"}], + ] + + +def test_fit_puts_the_stages_past_the_last_slot_into_one_job(): + stages = [[job("ros2-a")], [job("ros2-b")], [job("ros2-c"), job("ros2-d")], [job("ros2-e")]] + + fitted = pr_stages.fit(stages, slots=3) + + assert len(fitted) == 3 + assert [j["recipes"] for j in fitted[2]] == ["ros2-c ros2-d ros2-e"] + + +def test_fit_shortens_long_job_names(): + (fitted,) = pr_stages.fit([[{"name": "x" * 100, "recipes": "ros2-x"}]]) + assert len(fitted[0]["name"]) == pr_stages.MAX_NAME + + +def test_cached_recipes_need_every_variant_in_the_cache(monkeypatch, tmp_path): + recipes, cache = tmp_path / "recipes", tmp_path / "cache" + for name in ("ros2-a", "ros2-b", "ros2-c"): + (recipes / name).mkdir(parents=True) + (recipes / name / "recipe.yaml").write_text("{}\n") + (cache / "linux-64").mkdir(parents=True) + (cache / "noarch").mkdir() + (cache / "linux-64" / "ros2-a-1.0.0-py312_0.conda").touch() + (cache / "linux-64" / "ros2-a-1.0.0-py313_0.conda").touch() + (cache / "linux-64" / "ros2-b-1.0.0-py312_0.conda").touch() + (cache / "noarch" / "ros2-c-1.0.0-0.conda").touch() + monkeypatch.setattr(pr_stages.build_timings, "variant_builds", + lambda path, platform: {"ros2-a": 2, "ros2-b": 2}) + + assert pr_stages.cached_recipes(recipes, "linux-64", cache) == {"ros2-a", "ros2-c"} + + +def test_every_slot_waits_for_the_one_before(): + jobs = yaml.safe_load(pr_stages.workflow())["jobs"] + + assert jobs["stage_0"]["needs"] == "plan" + for k in range(1, pr_stages.SLOTS): + assert jobs[f"stage_{k}"]["needs"] == ["plan", f"stage_{k - 1}"] + assert f"> {k} }}" in jobs[f"stage_{k}"]["if"] + assert jobs["collect"]["needs"] == ["plan", *(f"stage_{k}" for k in range(pr_stages.SLOTS))] + + +def test_the_workflow_is_up_to_date(): + assert pr_stages.WORKFLOW.read_text() == pr_stages.workflow(), "python tools/pr_stages.py workflow" + + +def test_rattler_build_reads_the_local_channel(monkeypatch): + monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False) + monkeypatch.setenv("ROBOSTACK_LOCAL_CHANNEL", "pr-channel") + + cmd = rs.rattler_build("jazzy", []) + + assert cmd[cmd.index("-c") + 1] == (rs.ROOT / "pr-channel").as_uri() + + +def test_rattler_build_without_the_local_channel(monkeypatch): + monkeypatch.delenv("ROBOSTACK_LOCAL_CHANNEL", raising=False) + monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False) + + assert not any(c.startswith("file:") for c in rs.rattler_build("jazzy", [])) From 306d989ce95e62b9c30cd1240eef918e093ab680 Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Thu, 12 Nov 2026 17:51:38 -0700 Subject: [PATCH 2/6] One set of stage slots for every distribution and platform, for pull requests and releases staged-build.yaml (tools/staged_build.py) has 64 stage slots shared by all distributions and platforms instead of 64 per distribution and platform. Each job reads its recipes from the schedule, downloads only the packages of the jobs it needs (vinca-gha's needs), and fails when one of them failed, so the other distributions and platforms go on. Jobs that succeeded in an earlier attempt skip their build when later stages are re-run. release.yaml runs the same workflow with uploads, by hand while the build branches still build the releases. --- .github/workflows/pr-stages.yaml | 1225 -------------- .github/workflows/release.yaml | 65 + .../{pr-stage-job.yaml => stage-job.yaml} | 122 +- .github/workflows/staged-build.yaml | 1474 +++++++++++++++++ .github/workflows/testpr.yaml | 18 +- pixi.lock | 12 +- pixi.toml | 2 +- tools/maintenance.py | 6 +- tools/pr_stages.py | 359 ---- tools/robostack.py | 2 +- tools/staged_build.py | 515 ++++++ tools/tests/test_ci_build.py | 2 +- tools/tests/test_pr_stages.py | 78 - tools/tests/test_staged_build.py | 105 ++ 14 files changed, 2268 insertions(+), 1717 deletions(-) delete mode 100644 .github/workflows/pr-stages.yaml create mode 100644 .github/workflows/release.yaml rename .github/workflows/{pr-stage-job.yaml => stage-job.yaml} (55%) create mode 100644 .github/workflows/staged-build.yaml delete mode 100644 tools/pr_stages.py create mode 100644 tools/staged_build.py delete mode 100644 tools/tests/test_pr_stages.py create mode 100644 tools/tests/test_staged_build.py diff --git a/.github/workflows/pr-stages.yaml b/.github/workflows/pr-stages.yaml deleted file mode 100644 index 06035be2e..000000000 --- a/.github/workflows/pr-stages.yaml +++ /dev/null @@ -1,1225 +0,0 @@ -# Generated by tools/pr_stages.py (python tools/pr_stages.py workflow); `pixi run rs check` -# checks that it is up to date. -# -# One distribution's pull-request build on one platform, in the build branches' stages: -# the plan job generates the recipes and lists each stage's jobs (vinca-gha --json), -# each stage_ slot builds its stage's jobs side by side (pr-stage-job.yaml), and -# collect keeps the packages as the pull request's build cache. Packages pass from -# stage to stage as artifacts (pkgs---*), which the later jobs read as -# a local channel: no secrets, so pull requests from forks build the same way. -name: PR stages - -on: - workflow_call: - inputs: - distro: - required: true - type: string - platform: - required: true - type: string - os: - description: The platform's runner - required: true - type: string - -env: - PYTHONIOENCODING: utf-8 - DISTRO: ${{ inputs.distro }} - PLATFORM: ${{ inputs.platform }} - # while uploads go to a test channel, packages already there aren't built again - ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} - # the pull request's packages, /*.conda; restored and saved on Linux for every platform - CACHE_DIR: pr-cache - -jobs: - plan: - runs-on: ubuntu-latest - outputs: - count: ${{ steps.plan.outputs.count }} - stages: ${{ steps.plan.outputs.stages }} - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - # git auto-maintenance can leave a lock file in the git clones pixi caches - # (e.g. vinca's), which breaks every later run restoring that cache - - name: Disable git auto-maintenance - run: git config --global maintenance.auto false && git config --global gc.auto 0 - - uses: prefix-dev/setup-pixi@v0.11.0 - with: - locked: true # the lock file must match pixi.toml - pixi-version: v0.81.0 - cache-write: false # saved from main only (pixi-cache.yaml) - - name: Generate recipes - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - for attempt in 1 2 3; do - if pixi run rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi - if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi - echo "Recipe generation attempt $attempt failed; retrying..." >&2 - sleep 15 - done - - name: Upload the recipes - uses: actions/upload-artifact@v6 - with: - name: recipes-${{ inputs.distro }}-${{ inputs.platform }} - path: distros/${{ inputs.distro }}/work/recipes - if-no-files-found: ignore - overwrite: true - retention-days: 7 - - name: Restore build cache - uses: actions/cache/restore@v5 - with: - path: ${{ env.CACHE_DIR }} - key: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }} - restore-keys: | - pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}- - # Temporary rebuild controls live in distros//ci.yaml. - - name: Apply ci.yaml cache controls - run: | - for subdir in "$PLATFORM" noarch; do - mkdir -p "$CACHE_DIR/$subdir" - pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR/$subdir" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" - done - shopt -s nullglob - packages=("$CACHE_DIR"/*/*.conda) - echo "${#packages[@]} packages restored from the cache" - for package in "${packages[@]}"; do basename "$package"; done - # the stage jobs' download pattern always matches this artifact - echo "cache" > "$CACHE_DIR/manifest-cache.txt" - - name: Upload the cached packages - uses: actions/upload-artifact@v6 - with: - name: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-cache - path: ${{ env.CACHE_DIR }} - compression-level: 0 # .conda files are compressed - overwrite: true - retention-days: 3 - - name: Estimate the build time - run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR/$PLATFORM" --platform "$PLATFORM" - - name: Plan the stages - id: plan - run: pixi run python tools/pr_stages.py plan --distro "$DISTRO" --platform "$PLATFORM" --cache-dir "$CACHE_DIR" - - stage_0: - needs: plan - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 0 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[0] }} - name: 0 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_1: - needs: [plan, stage_0] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 1 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[1] }} - name: 1 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_2: - needs: [plan, stage_1] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 2 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[2] }} - name: 2 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_3: - needs: [plan, stage_2] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 3 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[3] }} - name: 3 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_4: - needs: [plan, stage_3] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 4 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[4] }} - name: 4 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_5: - needs: [plan, stage_4] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 5 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[5] }} - name: 5 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_6: - needs: [plan, stage_5] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 6 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[6] }} - name: 6 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_7: - needs: [plan, stage_6] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 7 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[7] }} - name: 7 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_8: - needs: [plan, stage_7] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 8 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[8] }} - name: 8 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_9: - needs: [plan, stage_8] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 9 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[9] }} - name: 9 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_10: - needs: [plan, stage_9] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 10 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[10] }} - name: 10 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_11: - needs: [plan, stage_10] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 11 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[11] }} - name: 11 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_12: - needs: [plan, stage_11] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 12 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[12] }} - name: 12 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_13: - needs: [plan, stage_12] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 13 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[13] }} - name: 13 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_14: - needs: [plan, stage_13] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 14 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[14] }} - name: 14 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_15: - needs: [plan, stage_14] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 15 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[15] }} - name: 15 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_16: - needs: [plan, stage_15] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 16 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[16] }} - name: 16 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_17: - needs: [plan, stage_16] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 17 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[17] }} - name: 17 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_18: - needs: [plan, stage_17] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 18 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[18] }} - name: 18 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_19: - needs: [plan, stage_18] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 19 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[19] }} - name: 19 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_20: - needs: [plan, stage_19] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 20 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[20] }} - name: 20 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_21: - needs: [plan, stage_20] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 21 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[21] }} - name: 21 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_22: - needs: [plan, stage_21] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 22 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[22] }} - name: 22 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_23: - needs: [plan, stage_22] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 23 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[23] }} - name: 23 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_24: - needs: [plan, stage_23] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 24 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[24] }} - name: 24 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_25: - needs: [plan, stage_24] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 25 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[25] }} - name: 25 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_26: - needs: [plan, stage_25] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 26 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[26] }} - name: 26 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_27: - needs: [plan, stage_26] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 27 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[27] }} - name: 27 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_28: - needs: [plan, stage_27] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 28 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[28] }} - name: 28 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_29: - needs: [plan, stage_28] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 29 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[29] }} - name: 29 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_30: - needs: [plan, stage_29] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 30 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[30] }} - name: 30 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_31: - needs: [plan, stage_30] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 31 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[31] }} - name: 31 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_32: - needs: [plan, stage_31] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 32 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[32] }} - name: 32 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_33: - needs: [plan, stage_32] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 33 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[33] }} - name: 33 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_34: - needs: [plan, stage_33] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 34 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[34] }} - name: 34 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_35: - needs: [plan, stage_34] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 35 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[35] }} - name: 35 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_36: - needs: [plan, stage_35] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 36 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[36] }} - name: 36 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_37: - needs: [plan, stage_36] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 37 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[37] }} - name: 37 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_38: - needs: [plan, stage_37] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 38 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[38] }} - name: 38 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_39: - needs: [plan, stage_38] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 39 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[39] }} - name: 39 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_40: - needs: [plan, stage_39] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 40 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[40] }} - name: 40 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_41: - needs: [plan, stage_40] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 41 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[41] }} - name: 41 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_42: - needs: [plan, stage_41] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 42 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[42] }} - name: 42 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_43: - needs: [plan, stage_42] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 43 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[43] }} - name: 43 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_44: - needs: [plan, stage_43] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 44 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[44] }} - name: 44 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_45: - needs: [plan, stage_44] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 45 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[45] }} - name: 45 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_46: - needs: [plan, stage_45] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 46 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[46] }} - name: 46 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_47: - needs: [plan, stage_46] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 47 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[47] }} - name: 47 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_48: - needs: [plan, stage_47] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 48 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[48] }} - name: 48 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_49: - needs: [plan, stage_48] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 49 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[49] }} - name: 49 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_50: - needs: [plan, stage_49] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 50 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[50] }} - name: 50 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_51: - needs: [plan, stage_50] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 51 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[51] }} - name: 51 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_52: - needs: [plan, stage_51] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 52 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[52] }} - name: 52 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_53: - needs: [plan, stage_52] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 53 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[53] }} - name: 53 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_54: - needs: [plan, stage_53] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 54 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[54] }} - name: 54 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_55: - needs: [plan, stage_54] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 55 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[55] }} - name: 55 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_56: - needs: [plan, stage_55] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 56 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[56] }} - name: 56 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_57: - needs: [plan, stage_56] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 57 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[57] }} - name: 57 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_58: - needs: [plan, stage_57] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 58 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[58] }} - name: 58 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_59: - needs: [plan, stage_58] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 59 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[59] }} - name: 59 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_60: - needs: [plan, stage_59] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 60 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[60] }} - name: 60 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_61: - needs: [plan, stage_60] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 61 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[61] }} - name: 61 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_62: - needs: [plan, stage_61] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 62 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[62] }} - name: 62 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - stage_63: - needs: [plan, stage_62] - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > 63 }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[63] }} - name: 63 ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - - # The packages of every stage as the pull request's build cache (a new push builds - # only what changed), then the ABI check of what the pull request built. - collect: - needs: [plan, stage_0, stage_1, stage_2, stage_3, stage_4, stage_5, stage_6, stage_7, stage_8, stage_9, stage_10, stage_11, stage_12, stage_13, stage_14, stage_15, stage_16, stage_17, stage_18, stage_19, stage_20, stage_21, stage_22, stage_23, stage_24, stage_25, stage_26, stage_27, stage_28, stage_29, stage_30, stage_31, stage_32, stage_33, stage_34, stage_35, stage_36, stage_37, stage_38, stage_39, stage_40, stage_41, stage_42, stage_43, stage_44, stage_45, stage_46, stage_47, stage_48, stage_49, stage_50, stage_51, stage_52, stage_53, stage_54, stage_55, stage_56, stage_57, stage_58, stage_59, stage_60, stage_61, stage_62, stage_63] - if: ${{ always() && needs.plan.result == 'success' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - # git auto-maintenance can leave a lock file in the git clones pixi caches - # (e.g. vinca's), which breaks every later run restoring that cache - - name: Disable git auto-maintenance - run: git config --global maintenance.auto false && git config --global gc.auto 0 - - uses: prefix-dev/setup-pixi@v0.11.0 - with: - locked: true # the lock file must match pixi.toml - pixi-version: v0.81.0 - cache-write: false # saved from main only (pixi-cache.yaml) - - uses: actions/download-artifact@v7 - with: - pattern: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-* - merge-multiple: true - path: ${{ env.CACHE_DIR }} - - name: See the packages - run: | - rm -f "$CACHE_DIR"/manifest-*.txt - shopt -s nullglob - packages=("$CACHE_DIR"/*/*.conda) - echo "${#packages[@]} packages" - for package in "${packages[@]}"; do basename "$package"; done - - name: Save build cache - id: save-cache - uses: actions/cache/save@v5 - with: - path: ${{ env.CACHE_DIR }} - key: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} - - # Only the newest cache of a pull request/distribution/platform is restored; the ones - # of earlier runs and attempts only push others out of the repository's 10 GB, so - # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. - - name: Drop this pull request's superseded build caches - if: ${{ steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - PREFIX: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}- - KEEP: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} - run: | - # cache/save reports success even on upload failure: verify the replacement exists. - if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \ - --jq "if any(.[]; .key == \"$KEEP\") then .[] | select(.key != \"$KEEP\") | .id else error(\"New cache not found\") end"); then - echo "::warning::Could not verify the new build cache; keeping all previous caches." - exit 0 - fi - while read -r id; do - [ -n "$id" ] || continue - pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" - done <<< "$ids" - - # Compare what the pull request built with the released builds, so single packages - # can be bumped without a full rebuild; it reports and never fails the build. - - name: ABI check against the released builds - id: abi-check - if: inputs.platform == 'linux-64' - continue-on-error: true - run: | - pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR/$PLATFORM" --summary "abi-check-$DISTRO.md" - cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" - - name: Upload the ABI check - if: ${{ steps.abi-check.outcome == 'success' }} - uses: actions/upload-artifact@v6 - with: - name: abi-check-${{ inputs.distro }} - path: abi-check-${{ inputs.distro }}.md - retention-days: 7 - - # without recipes to build there is no recipes artifact, and nothing to stage - - name: Prepare the work directory - if: needs.plan.outputs.count != '0' - run: pixi run rs "$DISTRO" prepare - - if: needs.plan.outputs.count != '0' - uses: actions/download-artifact@v7 - with: - name: recipes-${{ inputs.distro }}-${{ inputs.platform }} - path: distros/${{ inputs.distro }}/work/recipes - - name: Generate the staged build workflows (catches post-merge problems) - if: needs.plan.outputs.count != '0' - run: | - for attempt in 1 2 3; do - if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi - if [[ "$attempt" == 3 ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi - echo "Workflow generation attempt $attempt failed; retrying..." >&2 - sleep 15 - done diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..6c8e9990f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,65 @@ +# Release builds in vinca-gha's stages (staged-build.yaml), uploading to each +# distribution's channel (or the repository variable ROBOSTACK_UPLOAD_CHANNEL): the +# packages missing there are built and uploaded. Started by hand while the build +# branches (main.yaml) still build the releases. +# +# prefix.dev uploads use trusted publishing: the channels' Repository Access must allow +# .github/workflows/stage-job.yaml (called from this workflow). +name: Release build + +on: + workflow_dispatch: + inputs: + distros: + description: Space-separated distributions (default all) + required: false + platforms: + description: Space-separated platforms, or linux / osx / win (default all) + required: false + +concurrency: + group: release-build + cancel-in-progress: false + +permissions: + contents: read + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + distros: ${{ steps.plan.outputs.distros }} + targets: ${{ steps.plan.outputs.targets }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - id: plan + env: + DISTROS: ${{ inputs.distros }} + PLATFORMS: ${{ inputs.platforms }} + run: | + # the same selection as a pull request's build: / build: labels + labels=$(pixi run python -c 'import json,os; print(json.dumps([f"build:{n}" for n in (os.environ["DISTROS"] + " " + os.environ["PLATFORMS"]).split()]))') + echo "distros=$(pixi run python tools/robostack.py changed-distros --base HEAD --all --labels "$labels")" | tee -a "$GITHUB_OUTPUT" + echo "targets=$(pixi run python tools/robostack.py build-targets --labels "$labels")" | tee -a "$GITHUB_OUTPUT" + + build: + needs: plan + if: needs.plan.outputs.distros != '[]' + permissions: + contents: read + id-token: write # prefix.dev trusted publishing + uses: ./.github/workflows/staged-build.yaml + secrets: inherit # ANACONDA_API_TOKEN_ + with: + distros: ${{ needs.plan.outputs.distros }} + targets: ${{ needs.plan.outputs.targets }} + upload: true diff --git a/.github/workflows/pr-stage-job.yaml b/.github/workflows/stage-job.yaml similarity index 55% rename from .github/workflows/pr-stage-job.yaml rename to .github/workflows/stage-job.yaml index f32ece0a3..230073800 100644 --- a/.github/workflows/pr-stage-job.yaml +++ b/.github/workflows/stage-job.yaml @@ -1,7 +1,8 @@ -# One build job of a staged pull-request build (pr-stages.yaml): builds its recipes with -# the packages of the earlier stages (and the restored cache) as a local channel, and -# passes what it built on to the later stages and the cache as an artifact. -name: PR stage job +# One build job of a staged build (staged-build.yaml, tools/staged_build.py): builds its +# recipes with the packages of the jobs it needs (and the restored cache) as a local +# channel, and passes what it built on as an artifact, pkgs---, +# with status/; releases also upload it to the distribution's channels. +name: Stage job on: workflow_call: @@ -15,12 +16,15 @@ on: os: required: true type: string - recipes: - description: Space-separated recipes + job: + description: The job's id in the schedule (vinca-gha's stage__job_) required: true type: string - artifact: - description: Name of the artifact with the packages the job builds + upload: + required: true + type: boolean + token_secret: + description: The repository secret with the distribution's anaconda.org token required: true type: string @@ -28,8 +32,8 @@ env: PYTHONIOENCODING: utf-8 DISTRO: ${{ inputs.distro }} PLATFORM: ${{ inputs.platform }} - RECIPES: ${{ inputs.recipes }} - ARTIFACT: ${{ inputs.artifact }} + JOB: ${{ inputs.job }} + ARTIFACT: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ inputs.job }} ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} # rattler_build() in tools/robostack.py adds it as a channel (relative to the repository) ROBOSTACK_LOCAL_CHANNEL: pr-channel @@ -41,27 +45,48 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 350 steps: + # Re-running failed jobs re-runs every later stage too: the jobs that succeeded + # before keep their packages. + - uses: actions/download-artifact@v7 + with: + pattern: ${{ env.ARTIFACT }} + merge-multiple: true + path: earlier + - name: Built in an earlier attempt? + id: earlier + shell: bash + run: | + if [ "$(cat "earlier/status/$JOB" 2>/dev/null)" == success ]; then + echo "Built in an earlier attempt of this run" + echo "built=true" >> "$GITHUB_OUTPUT" + fi + + # (clears the directory, earlier/ included: it isn't a repository yet) - uses: actions/checkout@v6 + if: steps.earlier.outputs.built != 'true' with: persist-credentials: false # git auto-maintenance can leave a lock file in the git clones pixi caches # (e.g. vinca's), which breaks every later run restoring that cache - name: Disable git auto-maintenance + if: steps.earlier.outputs.built != 'true' + shell: bash run: git config --global maintenance.auto false && git config --global gc.auto 0 && git config --global core.longpaths true # pixi's package cache on D:, with the builds' (see the Windows step below) - name: "pixi's cache on D:" - if: runner.os == 'Windows' + if: steps.earlier.outputs.built != 'true' && runner.os == 'Windows' shell: pwsh run: '"PIXI_CACHE_DIR=D:\pixi-cache" | Out-File -FilePath $env:GITHUB_ENV -Append' - uses: prefix-dev/setup-pixi@v0.11.0 + if: steps.earlier.outputs.built != 'true' with: locked: true # the lock file must match pixi.toml pixi-version: v0.81.0 cache-write: false # saved from main only (pixi-cache.yaml) - name: Windows workarounds (short build path, long paths, PATH and SDK clean-up) - if: runner.os == 'Windows' + if: steps.earlier.outputs.built != 'true' && runner.os == 'Windows' shell: pwsh run: | # Defender's real-time scan slows down installing and deleting the environments' @@ -79,44 +104,63 @@ jobs: # Spotlight indexes every file the environments and builds write, using up to a # core or two of the runner's three or four; the runner is thrown away after the job - name: Turn off Spotlight - if: runner.os == 'macOS' + if: steps.earlier.outputs.built != 'true' && runner.os == 'macOS' run: sudo mdutil -a -i off || true - name: Clean up PATH - if: runner.os == 'Windows' + if: steps.earlier.outputs.built != 'true' && runner.os == 'Windows' uses: egor-tensin/cleanup-path@v5 with: dirs: 'C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin' - - name: Disable git auto-maintenance in source caches + - uses: actions/download-artifact@v7 + if: steps.earlier.outputs.built != 'true' + with: + name: schedule + - name: The job's recipes and requirements + if: steps.earlier.outputs.built != 'true' shell: bash -l {0} - run: git config --global maintenance.auto false + run: pixi run python tools/staged_build.py job --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" --job "$JOB" - name: Prepare the work directory + if: steps.earlier.outputs.built != 'true' shell: bash -l {0} run: pixi run rs "$DISTRO" prepare # the plan job's recipes (generated once for all the jobs) - uses: actions/download-artifact@v7 + if: steps.earlier.outputs.built != 'true' with: name: recipes-${{ inputs.distro }}-${{ inputs.platform }} path: distros/${{ inputs.distro }}/work/recipes - # everything the earlier stages built, and the restored cache + # the packages of the jobs this one needs, and the restored cache - uses: actions/download-artifact@v7 + if: steps.earlier.outputs.built != 'true' with: - pattern: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-* + pattern: ${{ env.NEEDS_PATTERN }} merge-multiple: true path: pr-channel - - name: Index the earlier stages' packages + - name: Check and index the packages of the jobs this one needs + if: steps.earlier.outputs.built != 'true' shell: bash -l {0} run: | + failed=() + for job in $NEEDED; do + [ "$(cat "pr-channel/status/$job" 2>/dev/null)" == success ] || failed+=("$job") + done + if (( ${#failed[@]} )); then + echo "::error::Not built, which this job needs: ${failed[*]} (see their jobs)" + exit 1 + fi + rm -rf pr-channel/status mkdir -p "pr-channel/$PLATFORM" pr-channel/noarch shopt -s nullglob packages=(pr-channel/*/*.conda) - echo "${#packages[@]} packages from the earlier stages and the cache" + echo "${#packages[@]} packages from the jobs this one needs and the cache" pixi run rattler-index fs pr-channel --force - name: Build recipes id: build-recipes + if: steps.earlier.outputs.built != 'true' shell: bash -l {0} run: | set +e @@ -129,41 +173,57 @@ jobs: exit "${PIPESTATUS[0]}" - name: Summarize build result - if: always() + if: ${{ always() && steps.build-recipes.outcome != 'skipped' }} shell: bash -l {0} run: | pixi run python tools/summarize_rattler_build.py --log rattler-build.log \ --platform "$DISTRO $PLATFORM" --outcome "${{ steps.build-recipes.outcome }}" >> "$GITHUB_STEP_SUMMARY" - # also after a failure: what was built goes into the cache + # also after a failure: what was built goes into the pull request's cache, and the + # status tells the later jobs - name: Collect the packages - if: ${{ success() || failure() }} + if: ${{ (success() || failure()) && steps.earlier.outputs.built != 'true' }} shell: bash -l {0} + env: + OUTCOME: ${{ steps.build-recipes.outcome }} run: | shopt -s nullglob - mkdir -p pkgs + mkdir -p pkgs/status for subdir in "$PLATFORM" noarch; do files=("$OUTPUT_DIR/$subdir"/*.conda) if (( ${#files[@]} )); then mkdir -p "pkgs/$subdir" && cp "${files[@]}" "pkgs/$subdir/"; fi done - # keeps the / directories in the artifact (it starts at the files' common directory) - (cd pkgs && printf '%s\n' ./*/*.conda) > "pkgs/manifest-$ARTIFACT.txt" - cat "pkgs/manifest-$ARTIFACT.txt" + echo "$OUTCOME" > "pkgs/status/$JOB" + (cd pkgs && printf '%s\n' ./*/*.conda) - name: Upload the packages - if: ${{ success() || failure() }} + if: ${{ (success() || failure()) && steps.earlier.outputs.built != 'true' }} uses: actions/upload-artifact@v6 with: - name: ${{ inputs.artifact }} + name: ${{ env.ARTIFACT }} path: pkgs compression-level: 0 # .conda files are compressed overwrite: true # re-runs of a failed job retention-days: 3 + # prefix.dev: trusted publishing (the caller grants id-token: write); anaconda.org: + # the distribution's token. Both skip packages that are already there. + - name: Upload to the distribution's channels + if: ${{ inputs.upload && steps.build-recipes.outcome == 'success' }} + shell: bash -l {0} + env: + ANACONDA_API_TOKEN: ${{ secrets[inputs.token_secret] }} + run: | + shopt -s nullglob + for package in pkgs/*/*.conda; do + # the upload task runs in distros//work + pixi run rs "$DISTRO" upload "../../../$package" + done + - name: Upload build log - if: always() + if: ${{ always() && steps.build-recipes.outcome != 'skipped' }} uses: actions/upload-artifact@v6 with: - name: build-log-${{ inputs.artifact }}-${{ github.run_attempt }} + name: build-log-${{ inputs.distro }}-${{ inputs.platform }}-${{ inputs.job }}-${{ github.run_attempt }} path: rattler-build.log if-no-files-found: warn retention-days: 7 diff --git a/.github/workflows/staged-build.yaml b/.github/workflows/staged-build.yaml new file mode 100644 index 000000000..16f7e5aac --- /dev/null +++ b/.github/workflows/staged-build.yaml @@ -0,0 +1,1474 @@ +# Generated by tools/staged_build.py (python tools/staged_build.py workflow); `pixi run rs check` +# checks that it is up to date. +# +# The given distributions and platforms, built in vinca-gha's stages, one job per batch +# of recipes (stage-job.yaml): pull requests (testpr.yaml) keep the packages as the +# pull request's build cache, releases (release.yaml) upload them. See +# tools/staged_build.py for how the stages become jobs. +name: Staged build + +on: + workflow_call: + inputs: + distros: + description: JSON list of distributions + required: true + type: string + targets: + description: 'JSON list of platforms and their runners, [{"os": ..., "platform": ...}]' + required: true + type: string + upload: + description: Upload the packages to the distributions' channels (else keep them as the pull request's cache) + required: true + type: boolean + +env: + PYTHONIOENCODING: utf-8 + # while uploads go to a test channel, packages already there aren't built again + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} + # the pull request's packages, /*.conda; restored and saved on Linux for every platform + CACHE_DIR: pr-cache + +jobs: + plan: + strategy: + fail-fast: false + matrix: + distro: ${{ fromJSON(inputs.distros) }} + target: ${{ fromJSON(inputs.targets) }} + name: plan ${{ matrix.distro }} ${{ matrix.target.platform }} + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.target.platform }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - name: Generate recipes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi + echo "Recipe generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done + - name: Upload the recipes + uses: actions/upload-artifact@v6 + with: + name: recipes-${{ matrix.distro }}-${{ matrix.target.platform }} + path: distros/${{ matrix.distro }}/work/recipes + if-no-files-found: ignore + overwrite: true + retention-days: 7 + - name: Restore build cache + if: ${{ !inputs.upload }} + uses: actions/cache/restore@v5 + with: + path: ${{ env.CACHE_DIR }} + key: staged-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }} + restore-keys: | + staged-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}- + # Temporary rebuild controls live in distros//ci.yaml. + - name: Apply ci.yaml cache controls + run: | + for subdir in "$PLATFORM" noarch; do + mkdir -p "$CACHE_DIR/$subdir" + pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR/$subdir" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" + done + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages restored from the cache" + for package in "${packages[@]}"; do basename "$package"; done + mkdir -p "$CACHE_DIR/status" && echo success > "$CACHE_DIR/status/cache" + - name: Upload the cached packages + uses: actions/upload-artifact@v6 + with: + name: pkgs-${{ matrix.distro }}-${{ matrix.target.platform }}-cache + path: ${{ env.CACHE_DIR }} + compression-level: 0 # .conda files are compressed + overwrite: true + retention-days: 3 + - name: Estimate the build time + run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR/$PLATFORM" --platform "$PLATFORM" + - name: Plan the stages + env: + OS: ${{ matrix.target.os }} + run: pixi run python tools/staged_build.py plan --distro "$DISTRO" --platform "$PLATFORM" --os "$OS" --cache-dir "$CACHE_DIR" --out "plan-$DISTRO-$PLATFORM.json" + - uses: actions/upload-artifact@v6 + with: + name: plan-${{ matrix.distro }}-${{ matrix.target.platform }} + path: plan-${{ matrix.distro }}-${{ matrix.target.platform }}.json + overwrite: true + retention-days: 7 + + # Every plan in the stage slots, and the schedule the stage jobs read their recipes from; + # a failed plan leaves out only its distribution and platform. + schedule: + needs: plan + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + outputs: + count: ${{ steps.schedule.outputs.count }} + combos: ${{ steps.schedule.outputs.combos }} + slot_0: ${{ steps.schedule.outputs.slot_0 }} + slot_1: ${{ steps.schedule.outputs.slot_1 }} + slot_2: ${{ steps.schedule.outputs.slot_2 }} + slot_3: ${{ steps.schedule.outputs.slot_3 }} + slot_4: ${{ steps.schedule.outputs.slot_4 }} + slot_5: ${{ steps.schedule.outputs.slot_5 }} + slot_6: ${{ steps.schedule.outputs.slot_6 }} + slot_7: ${{ steps.schedule.outputs.slot_7 }} + slot_8: ${{ steps.schedule.outputs.slot_8 }} + slot_9: ${{ steps.schedule.outputs.slot_9 }} + slot_10: ${{ steps.schedule.outputs.slot_10 }} + slot_11: ${{ steps.schedule.outputs.slot_11 }} + slot_12: ${{ steps.schedule.outputs.slot_12 }} + slot_13: ${{ steps.schedule.outputs.slot_13 }} + slot_14: ${{ steps.schedule.outputs.slot_14 }} + slot_15: ${{ steps.schedule.outputs.slot_15 }} + slot_16: ${{ steps.schedule.outputs.slot_16 }} + slot_17: ${{ steps.schedule.outputs.slot_17 }} + slot_18: ${{ steps.schedule.outputs.slot_18 }} + slot_19: ${{ steps.schedule.outputs.slot_19 }} + slot_20: ${{ steps.schedule.outputs.slot_20 }} + slot_21: ${{ steps.schedule.outputs.slot_21 }} + slot_22: ${{ steps.schedule.outputs.slot_22 }} + slot_23: ${{ steps.schedule.outputs.slot_23 }} + slot_24: ${{ steps.schedule.outputs.slot_24 }} + slot_25: ${{ steps.schedule.outputs.slot_25 }} + slot_26: ${{ steps.schedule.outputs.slot_26 }} + slot_27: ${{ steps.schedule.outputs.slot_27 }} + slot_28: ${{ steps.schedule.outputs.slot_28 }} + slot_29: ${{ steps.schedule.outputs.slot_29 }} + slot_30: ${{ steps.schedule.outputs.slot_30 }} + slot_31: ${{ steps.schedule.outputs.slot_31 }} + slot_32: ${{ steps.schedule.outputs.slot_32 }} + slot_33: ${{ steps.schedule.outputs.slot_33 }} + slot_34: ${{ steps.schedule.outputs.slot_34 }} + slot_35: ${{ steps.schedule.outputs.slot_35 }} + slot_36: ${{ steps.schedule.outputs.slot_36 }} + slot_37: ${{ steps.schedule.outputs.slot_37 }} + slot_38: ${{ steps.schedule.outputs.slot_38 }} + slot_39: ${{ steps.schedule.outputs.slot_39 }} + slot_40: ${{ steps.schedule.outputs.slot_40 }} + slot_41: ${{ steps.schedule.outputs.slot_41 }} + slot_42: ${{ steps.schedule.outputs.slot_42 }} + slot_43: ${{ steps.schedule.outputs.slot_43 }} + slot_44: ${{ steps.schedule.outputs.slot_44 }} + slot_45: ${{ steps.schedule.outputs.slot_45 }} + slot_46: ${{ steps.schedule.outputs.slot_46 }} + slot_47: ${{ steps.schedule.outputs.slot_47 }} + slot_48: ${{ steps.schedule.outputs.slot_48 }} + slot_49: ${{ steps.schedule.outputs.slot_49 }} + slot_50: ${{ steps.schedule.outputs.slot_50 }} + slot_51: ${{ steps.schedule.outputs.slot_51 }} + slot_52: ${{ steps.schedule.outputs.slot_52 }} + slot_53: ${{ steps.schedule.outputs.slot_53 }} + slot_54: ${{ steps.schedule.outputs.slot_54 }} + slot_55: ${{ steps.schedule.outputs.slot_55 }} + slot_56: ${{ steps.schedule.outputs.slot_56 }} + slot_57: ${{ steps.schedule.outputs.slot_57 }} + slot_58: ${{ steps.schedule.outputs.slot_58 }} + slot_59: ${{ steps.schedule.outputs.slot_59 }} + slot_60: ${{ steps.schedule.outputs.slot_60 }} + slot_61: ${{ steps.schedule.outputs.slot_61 }} + slot_62: ${{ steps.schedule.outputs.slot_62 }} + slot_63: ${{ steps.schedule.outputs.slot_63 }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - uses: actions/download-artifact@v7 + with: + pattern: plan-* + merge-multiple: true + path: plans + - name: Schedule the stages + id: schedule + run: pixi run python tools/staged_build.py schedule --out schedule.json plans/*.json + - uses: actions/upload-artifact@v6 + with: + name: schedule + path: schedule.json + overwrite: true + retention-days: 7 + + stage_0: + needs: schedule + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 0 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_0) }} + name: 0 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_1: + needs: [schedule, stage_0] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 1 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_1) }} + name: 1 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_2: + needs: [schedule, stage_1] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 2 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_2) }} + name: 2 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_3: + needs: [schedule, stage_2] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 3 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_3) }} + name: 3 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_4: + needs: [schedule, stage_3] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 4 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_4) }} + name: 4 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_5: + needs: [schedule, stage_4] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 5 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_5) }} + name: 5 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_6: + needs: [schedule, stage_5] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 6 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_6) }} + name: 6 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_7: + needs: [schedule, stage_6] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 7 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_7) }} + name: 7 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_8: + needs: [schedule, stage_7] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 8 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_8) }} + name: 8 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_9: + needs: [schedule, stage_8] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 9 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_9) }} + name: 9 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_10: + needs: [schedule, stage_9] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 10 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_10) }} + name: 10 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_11: + needs: [schedule, stage_10] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 11 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_11) }} + name: 11 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_12: + needs: [schedule, stage_11] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 12 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_12) }} + name: 12 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_13: + needs: [schedule, stage_12] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 13 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_13) }} + name: 13 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_14: + needs: [schedule, stage_13] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 14 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_14) }} + name: 14 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_15: + needs: [schedule, stage_14] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 15 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_15) }} + name: 15 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_16: + needs: [schedule, stage_15] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 16 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_16) }} + name: 16 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_17: + needs: [schedule, stage_16] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 17 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_17) }} + name: 17 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_18: + needs: [schedule, stage_17] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 18 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_18) }} + name: 18 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_19: + needs: [schedule, stage_18] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 19 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_19) }} + name: 19 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_20: + needs: [schedule, stage_19] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 20 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_20) }} + name: 20 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_21: + needs: [schedule, stage_20] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 21 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_21) }} + name: 21 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_22: + needs: [schedule, stage_21] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 22 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_22) }} + name: 22 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_23: + needs: [schedule, stage_22] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 23 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_23) }} + name: 23 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_24: + needs: [schedule, stage_23] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 24 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_24) }} + name: 24 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_25: + needs: [schedule, stage_24] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 25 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_25) }} + name: 25 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_26: + needs: [schedule, stage_25] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 26 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_26) }} + name: 26 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_27: + needs: [schedule, stage_26] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 27 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_27) }} + name: 27 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_28: + needs: [schedule, stage_27] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 28 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_28) }} + name: 28 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_29: + needs: [schedule, stage_28] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 29 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_29) }} + name: 29 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_30: + needs: [schedule, stage_29] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 30 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_30) }} + name: 30 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_31: + needs: [schedule, stage_30] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 31 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_31) }} + name: 31 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_32: + needs: [schedule, stage_31] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 32 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_32) }} + name: 32 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_33: + needs: [schedule, stage_32] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 33 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_33) }} + name: 33 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_34: + needs: [schedule, stage_33] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 34 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_34) }} + name: 34 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_35: + needs: [schedule, stage_34] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 35 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_35) }} + name: 35 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_36: + needs: [schedule, stage_35] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 36 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_36) }} + name: 36 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_37: + needs: [schedule, stage_36] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 37 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_37) }} + name: 37 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_38: + needs: [schedule, stage_37] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 38 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_38) }} + name: 38 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_39: + needs: [schedule, stage_38] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 39 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_39) }} + name: 39 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_40: + needs: [schedule, stage_39] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 40 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_40) }} + name: 40 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_41: + needs: [schedule, stage_40] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 41 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_41) }} + name: 41 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_42: + needs: [schedule, stage_41] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 42 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_42) }} + name: 42 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_43: + needs: [schedule, stage_42] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 43 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_43) }} + name: 43 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_44: + needs: [schedule, stage_43] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 44 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_44) }} + name: 44 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_45: + needs: [schedule, stage_44] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 45 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_45) }} + name: 45 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_46: + needs: [schedule, stage_45] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 46 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_46) }} + name: 46 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_47: + needs: [schedule, stage_46] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 47 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_47) }} + name: 47 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_48: + needs: [schedule, stage_47] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 48 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_48) }} + name: 48 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_49: + needs: [schedule, stage_48] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 49 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_49) }} + name: 49 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_50: + needs: [schedule, stage_49] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 50 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_50) }} + name: 50 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_51: + needs: [schedule, stage_50] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 51 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_51) }} + name: 51 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_52: + needs: [schedule, stage_51] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 52 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_52) }} + name: 52 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_53: + needs: [schedule, stage_52] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 53 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_53) }} + name: 53 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_54: + needs: [schedule, stage_53] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 54 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_54) }} + name: 54 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_55: + needs: [schedule, stage_54] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 55 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_55) }} + name: 55 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_56: + needs: [schedule, stage_55] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 56 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_56) }} + name: 56 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_57: + needs: [schedule, stage_56] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 57 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_57) }} + name: 57 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_58: + needs: [schedule, stage_57] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 58 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_58) }} + name: 58 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_59: + needs: [schedule, stage_58] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 59 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_59) }} + name: 59 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_60: + needs: [schedule, stage_59] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 60 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_60) }} + name: 60 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_61: + needs: [schedule, stage_60] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 61 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_61) }} + name: 61 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_62: + needs: [schedule, stage_61] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 62 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_62) }} + name: 62 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_63: + needs: [schedule, stage_62] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 63 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_63) }} + name: 63 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + # Pull requests: the packages of every stage as the pull request's build cache (a new + # push builds only what changed), then the ABI check of what the pull request built. + collect: + needs: [schedule, stage_0, stage_1, stage_2, stage_3, stage_4, stage_5, stage_6, stage_7, stage_8, stage_9, stage_10, stage_11, stage_12, stage_13, stage_14, stage_15, stage_16, stage_17, stage_18, stage_19, stage_20, stage_21, stage_22, stage_23, stage_24, stage_25, stage_26, stage_27, stage_28, stage_29, stage_30, stage_31, stage_32, stage_33, stage_34, stage_35, stage_36, stage_37, stage_38, stage_39, stage_40, stage_41, stage_42, stage_43, stage_44, stage_45, stage_46, stage_47, stage_48, stage_49, stage_50, stage_51, stage_52, stage_53, stage_54, stage_55, stage_56, stage_57, stage_58, stage_59, stage_60, stage_61, stage_62, stage_63] + if: ${{ always() && !inputs.upload && needs.schedule.result == 'success' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.combos) }} + name: collect ${{ matrix.distro }} ${{ matrix.platform }} + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - uses: actions/download-artifact@v7 + with: + pattern: pkgs-${{ matrix.distro }}-${{ matrix.platform }}-* + merge-multiple: true + path: ${{ env.CACHE_DIR }} + - name: See the packages + run: | + rm -rf "$CACHE_DIR/status" + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages" + for package in "${packages[@]}"; do basename "$package"; done + - name: Save build cache + id: save-cache + uses: actions/cache/save@v5 + with: + path: ${{ env.CACHE_DIR }} + key: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + + # Only the newest cache of a pull request/distribution/platform is restored; the ones + # of earlier runs and attempts only push others out of the repository's 10 GB, so + # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. + - name: Drop this pull request's superseded build caches + if: ${{ steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PREFIX: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}- + KEEP: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + run: | + # cache/save reports success even on upload failure: verify the replacement exists. + if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \ + --jq "if any(.[]; .key == \"$KEEP\") then .[] | select(.key != \"$KEEP\") | .id else error(\"New cache not found\") end"); then + echo "::warning::Could not verify the new build cache; keeping all previous caches." + exit 0 + fi + while read -r id; do + [ -n "$id" ] || continue + pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" + done <<< "$ids" + + # Compare what the pull request built with the released builds, so single packages + # can be bumped without a full rebuild; it reports and never fails the build. + - name: ABI check against the released builds + id: abi-check + if: matrix.platform == 'linux-64' + continue-on-error: true + run: | + pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR/$PLATFORM" --summary "abi-check-$DISTRO.md" + cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" + - name: Upload the ABI check + if: ${{ steps.abi-check.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: abi-check-${{ matrix.distro }} + path: abi-check-${{ matrix.distro }}.md + retention-days: 7 + + # the build branches' workflows, while they still build the releases; without + # recipes to build there is no recipes artifact, and nothing to stage + - name: Prepare the work directory + run: pixi run rs "$DISTRO" prepare + # a pattern, which (unlike a name) may match nothing + - uses: actions/download-artifact@v7 + with: + pattern: recipes-${{ matrix.distro }}-${{ matrix.platform }} + merge-multiple: true + path: distros/${{ matrix.distro }}/work/recipes + - name: Generate the staged build workflows (catches post-merge problems) + run: | + if [ -z "$(ls "distros/$DISTRO/work/recipes" 2>/dev/null)" ]; then echo "Nothing to build"; exit 0; fi + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi + echo "Workflow generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index ea289d632..bcaa57afc 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -136,25 +136,19 @@ jobs: pixi run rs "$DISTRO" check-orphaned-patches pixi run rs "$DISTRO" check-patches - # Each distribution and platform in the build branches' stages, one job per batch of - # recipes (pr-stages.yaml, generated by tools/pr_stages.py). + # The changed distributions on every platform, in vinca-gha's stages, one job per + # batch of recipes (staged-build.yaml, generated by tools/staged_build.py). build: permissions: actions: write # drop this pull request's superseded build caches contents: read needs: plan if: needs.plan.outputs.distros != '[]' - strategy: - fail-fast: false - matrix: - distro: ${{ fromJSON(needs.plan.outputs.distros) }} - target: ${{ fromJSON(needs.plan.outputs.targets) }} # tools/robostack.py BUILD_TARGETS - name: ${{ matrix.distro }} ${{ matrix.target.platform }} - uses: ./.github/workflows/pr-stages.yaml + uses: ./.github/workflows/staged-build.yaml with: - distro: ${{ matrix.distro }} - platform: ${{ matrix.target.platform }} - os: ${{ matrix.target.os }} + distros: ${{ needs.plan.outputs.distros }} + targets: ${{ needs.plan.outputs.targets }} # tools/robostack.py BUILD_TARGETS + upload: false # The ABI checks of all distributions as one pull-request comment, updated on every # push. Pull requests from forks get a read-only token, so the job is skipped for them: diff --git a/pixi.lock b/pixi.lock index 941590414..de1b72f0a 100644 --- a/pixi.lock +++ b/pixi.lock @@ -90,7 +90,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -170,7 +170,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -243,7 +243,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/tk-8.6.13-ha6c374e_4.conda - conda: https://prefix.dev/conda-forge/osx-64/yaml-0.2.5-had63097_3.conda - conda: https://prefix.dev/conda-forge/osx-64/zstd-1.5.7-hbc1a06c_7.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -316,7 +316,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - conda: https://prefix.dev/conda-forge/osx-arm64/yaml-0.2.5-h74c22ad_3.conda - conda: https://prefix.dev/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -389,7 +389,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/vcomp14-14.51.36247-habf1de7_41.conda - conda: https://prefix.dev/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://prefix.dev/conda-forge/win-64/zstd-1.5.7-h534d264_7.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -3289,7 +3289,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 size: 387535 timestamp: 1786599623274 -- pypi: git+https://github.com/EzraBrooks/vinca.git?rev=b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854#b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854 +- pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa name: vinca version: 0.2.0 requires_dist: diff --git a/pixi.toml b/pixi.toml index 8cafeac9e..b1946a7c8 100644 --- a/pixi.toml +++ b/pixi.toml @@ -36,7 +36,7 @@ git = "*" [pypi-dependencies] # One vinca for all distributions. -vinca = { git = "https://github.com/EzraBrooks/vinca.git", rev = "b7c7f3f96adfdc0d3f8e9c49c50d2375055a2854" } +vinca = { git = "https://github.com/EzraBrooks/vinca.git", rev = "0a04e395f01c70322f234b582422991ab6928caa" } # For local development, temporarily use (and regenerate pixi.lock): # vinca = { path = "../vinca", editable = true } diff --git a/tools/maintenance.py b/tools/maintenance.py index 1a2ee5519..5ad765f2b 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -1403,10 +1403,10 @@ def check() -> Result: text = wf.read_text() if "\non:" not in "\n" + text or "\ntrue:" in "\n" + text: problems.append(f"{wf.name}: no top-level `on:` trigger") - import pr_stages + import staged_build - if not pr_stages.WORKFLOW.is_file() or pr_stages.WORKFLOW.read_text() != pr_stages.workflow(): - problems.append(f"{pr_stages.WORKFLOW.relative_to(rs.ROOT)} is out of date (python tools/pr_stages.py workflow)") + if not staged_build.WORKFLOW.is_file() or staged_build.WORKFLOW.read_text() != staged_build.workflow(): + problems.append(f"{staged_build.WORKFLOW.relative_to(rs.ROOT)} is out of date (python tools/staged_build.py workflow)") sort = subprocess.run(["pixi", "run", "sort", "--check"], cwd=rs.ROOT, capture_output=True, text=True) if sort.returncode: problems.append("YAML files are not sorted (pixi run sort):\n" + tail(sort.stdout + sort.stderr, 10)) diff --git a/tools/pr_stages.py b/tools/pr_stages.py deleted file mode 100644 index 9abb4a6e4..000000000 --- a/tools/pr_stages.py +++ /dev/null @@ -1,359 +0,0 @@ -"""The staged pull-request build: the build-branch stages (vinca-gha) as matrix jobs. - - python tools/pr_stages.py plan --distro --platform

--cache-dir

- python tools/pr_stages.py workflow [--check] - -GitHub can't add jobs to a running workflow, so .github/workflows/pr-stages.yaml has a -fixed number of stage slots (SLOTS), stage_0 ... stage_, each after the one -before. `plan` (its plan job) lists each stage's jobs from `vinca-gha --json`, leaving -out the recipes whose every build is in the restored pull-request cache; a slot takes -its stage's jobs as its matrix and is skipped when there is no such stage. `workflow` -writes pr-stages.yaml (`rs check` checks that it is up to date). -""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import subprocess -import sys -from collections import Counter -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -import build_timings -import robostack as rs - -SLOTS = 64 # lyrical's full linux-64 rebuild has 47 stages at --batch_size 25 -MAX_NAME = 60 # job names are a stage's packages, which can be many -WORKFLOW = rs.ROOT / ".github" / "workflows" / "pr-stages.yaml" - - -def cached_recipes(recipes: Path, platform: str, cache_dir: Path) -> set[str]: - """The recipes whose every build (one per variant) is in the cache (/*.conda).""" - names = {p.parent.name for p in recipes.glob("*/recipe.yaml")} - variants = build_timings.variant_builds(recipes, platform) if names else {} - cached = Counter(f.name.rsplit("-", 2)[0] for f in cache_dir.glob("*/*.conda")) - return {r for r in names if cached[r] >= variants.get(r, 1)} - - -def fit(stages: list[list[dict]], slots: int = SLOTS) -> list[list[dict]]: - """The stages as at most `slots` stages: the ones past the last slot become one job - there (build-ci orders a job's recipes by their dependencies). Each job gets an id - for its artifact, stage and job number.""" - if len(stages) > slots: - rest = [job for stage in stages[slots - 1:] for job in stage] - tail = {"name": f"{len(rest)} jobs of stages {slots - 1}+", "recipes": " ".join(j["recipes"] for j in rest)} - stages = [*stages[:slots - 1], [tail]] - fitted = [] - for k, stage in enumerate(stages): - jobs = [] - for i, job in enumerate(stage): - name = job["name"] if len(job["name"]) <= MAX_NAME else job["name"][:MAX_NAME - 1] + "…" - jobs.append({"id": f"s{k}-{i}", "name": name, "recipes": job["recipes"]}) - fitted.append(jobs) - return fitted - - -def plan(distro: str, platform: str, cache_dir: Path, batch_size: int) -> list[list[dict]]: - w = rs.work_dir(distro) - recipes = w / "recipes" - cached = cached_recipes(recipes, platform, cache_dir) if recipes.is_dir() else set() - # vinca-gha stages what is in ./recipes; the cached recipes count as built, like the - # ones on the channel that generate-recipes -n leaves out - aside = w / "recipes-cached" - shutil.rmtree(aside, ignore_errors=True) - aside.mkdir() - for r in cached: - (recipes / r).rename(aside / r) - try: - out = w / "pr_stages.json" - cmd = ["vinca-gha", "--platform", platform, "-d", "./recipes", "--batch_size", str(batch_size), - "--json", out.name] - if build_timings.vinca_timings(distro, platform, w / "build_timings.json"): - cmd += ["--timings", "build_timings.json"] - if rs.run(cmd, w): - raise SystemExit("vinca-gha failed") - stages = json.loads(out.read_text())["stages"] - finally: - for r in cached: - (aside / r).rename(recipes / r) - aside.rmdir() - print(f"{len(cached)} recipes fully in the cache; {sum(map(len, stages))} jobs in {len(stages)} stages") - return fit(stages) - - -def workflow() -> str: - """pr-stages.yaml: HEADER, the stage slots, then COLLECT.""" - slots = [] - for k in range(SLOTS): - needs = "plan" if k == 0 else f"[plan, stage_{k - 1}]" - slots.append(SLOT.replace("@K@", str(k)).replace("@NEEDS@", needs)) - every = ", ".join(["plan", *(f"stage_{k}" for k in range(SLOTS))]) - return HEADER + "".join(slots) + COLLECT.replace("@NEEDS@", every) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - sub = parser.add_subparsers(dest="command", required=True) - p = sub.add_parser("plan", help="the stages of one distribution and platform, as GitHub outputs") - p.add_argument("--distro", required=True) - p.add_argument("--platform", required=True) - p.add_argument("--cache-dir", required=True, type=Path) - p.add_argument("--batch-size", type=int, default=25) - w = sub.add_parser("workflow", help=f"write {WORKFLOW.relative_to(rs.ROOT)}") - w.add_argument("--check", action="store_true", help="only check that it is up to date") - args = parser.parse_args() - - if args.command == "workflow": - text = workflow() - if args.check: - if not WORKFLOW.is_file() or WORKFLOW.read_text() != text: - print(f"{WORKFLOW.relative_to(rs.ROOT)} is out of date (python tools/pr_stages.py workflow)") - return 1 - return 0 - WORKFLOW.write_text(text) - return 0 - - stages = plan(args.distro, args.platform, args.cache_dir.resolve(), args.batch_size) - # the slots past the last stage are skipped, but GitHub reads their matrix anyway - padded = stages + [[{"id": "none", "name": "none", "recipes": ""}]] * (SLOTS - len(stages)) - lines = [f"count={len(stages)}", f"stages={json.dumps(padded, separators=(',', ':'))}"] - if output := os.environ.get("GITHUB_OUTPUT"): - with open(output, "a", encoding="utf-8") as fh: - fh.write("\n".join(lines) + "\n") - if summary := os.environ.get("GITHUB_STEP_SUMMARY"): - with open(summary, "a", encoding="utf-8") as fh: - jobs = sum(map(len, stages)) - fh.write(f"**{args.distro} {args.platform}**: {jobs} build jobs in {len(stages)} stages\n\n") - for k, stage in enumerate(stages): - print(f"stage {k}: " + "; ".join(job["recipes"] for job in stage)) - return 0 - - -SETUP = """\ - - uses: actions/checkout@v6 - with: - persist-credentials: false - # git auto-maintenance can leave a lock file in the git clones pixi caches - # (e.g. vinca's), which breaks every later run restoring that cache - - name: Disable git auto-maintenance - run: git config --global maintenance.auto false && git config --global gc.auto 0 - - uses: prefix-dev/setup-pixi@v0.11.0 - with: - locked: true # the lock file must match pixi.toml - pixi-version: v0.81.0 - cache-write: false # saved from main only (pixi-cache.yaml) -""" - -HEADER = """\ -# Generated by tools/pr_stages.py (python tools/pr_stages.py workflow); `pixi run rs check` -# checks that it is up to date. -# -# One distribution's pull-request build on one platform, in the build branches' stages: -# the plan job generates the recipes and lists each stage's jobs (vinca-gha --json), -# each stage_ slot builds its stage's jobs side by side (pr-stage-job.yaml), and -# collect keeps the packages as the pull request's build cache. Packages pass from -# stage to stage as artifacts (pkgs---*), which the later jobs read as -# a local channel: no secrets, so pull requests from forks build the same way. -name: PR stages - -on: - workflow_call: - inputs: - distro: - required: true - type: string - platform: - required: true - type: string - os: - description: The platform's runner - required: true - type: string - -env: - PYTHONIOENCODING: utf-8 - DISTRO: ${{ inputs.distro }} - PLATFORM: ${{ inputs.platform }} - # while uploads go to a test channel, packages already there aren't built again - ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} - # the pull request's packages, /*.conda; restored and saved on Linux for every platform - CACHE_DIR: pr-cache - -jobs: - plan: - runs-on: ubuntu-latest - outputs: - count: ${{ steps.plan.outputs.count }} - stages: ${{ steps.plan.outputs.stages }} - steps: -""" + SETUP + """\ - - name: Generate recipes - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - for attempt in 1 2 3; do - if pixi run rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi - if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi - echo "Recipe generation attempt $attempt failed; retrying..." >&2 - sleep 15 - done - - name: Upload the recipes - uses: actions/upload-artifact@v6 - with: - name: recipes-${{ inputs.distro }}-${{ inputs.platform }} - path: distros/${{ inputs.distro }}/work/recipes - if-no-files-found: ignore - overwrite: true - retention-days: 7 - - name: Restore build cache - uses: actions/cache/restore@v5 - with: - path: ${{ env.CACHE_DIR }} - key: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }} - restore-keys: | - pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}- - # Temporary rebuild controls live in distros//ci.yaml. - - name: Apply ci.yaml cache controls - run: | - for subdir in "$PLATFORM" noarch; do - mkdir -p "$CACHE_DIR/$subdir" - pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR/$subdir" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" - done - shopt -s nullglob - packages=("$CACHE_DIR"/*/*.conda) - echo "${#packages[@]} packages restored from the cache" - for package in "${packages[@]}"; do basename "$package"; done - # the stage jobs' download pattern always matches this artifact - echo "cache" > "$CACHE_DIR/manifest-cache.txt" - - name: Upload the cached packages - uses: actions/upload-artifact@v6 - with: - name: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-cache - path: ${{ env.CACHE_DIR }} - compression-level: 0 # .conda files are compressed - overwrite: true - retention-days: 3 - - name: Estimate the build time - run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR/$PLATFORM" --platform "$PLATFORM" - - name: Plan the stages - id: plan - run: pixi run python tools/pr_stages.py plan --distro "$DISTRO" --platform "$PLATFORM" --cache-dir "$CACHE_DIR" - -""" - -# a skipped slot doesn't skip the ones after it (!failure() instead of the default -# success()), but a failed one does: its packages are missing for the later stages -SLOT = """\ - stage_@K@: - needs: @NEEDS@ - if: ${{ !cancelled() && !failure() && fromJSON(needs.plan.outputs.count) > @K@ }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.plan.outputs.stages)[@K@] }} - name: @K@ ${{ matrix.name }} - uses: ./.github/workflows/pr-stage-job.yaml - with: - distro: ${{ inputs.distro }} - platform: ${{ inputs.platform }} - os: ${{ inputs.os }} - recipes: ${{ matrix.recipes }} - artifact: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ matrix.id }} - -""" - -COLLECT = """\ - # The packages of every stage as the pull request's build cache (a new push builds - # only what changed), then the ABI check of what the pull request built. - collect: - needs: [@NEEDS@] - if: ${{ always() && needs.plan.result == 'success' }} - runs-on: ubuntu-latest - steps: -""" + SETUP + """\ - - uses: actions/download-artifact@v7 - with: - pattern: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-* - merge-multiple: true - path: ${{ env.CACHE_DIR }} - - name: See the packages - run: | - rm -f "$CACHE_DIR"/manifest-*.txt - shopt -s nullglob - packages=("$CACHE_DIR"/*/*.conda) - echo "${#packages[@]} packages" - for package in "${packages[@]}"; do basename "$package"; done - - name: Save build cache - id: save-cache - uses: actions/cache/save@v5 - with: - path: ${{ env.CACHE_DIR }} - key: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} - - # Only the newest cache of a pull request/distribution/platform is restored; the ones - # of earlier runs and attempts only push others out of the repository's 10 GB, so - # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. - - name: Drop this pull request's superseded build caches - if: ${{ steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - PREFIX: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}- - KEEP: pr-stages-${{ inputs.distro }}-${{ inputs.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} - run: | - # cache/save reports success even on upload failure: verify the replacement exists. - if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \\ - --jq "if any(.[]; .key == \\"$KEEP\\") then .[] | select(.key != \\"$KEEP\\") | .id else error(\\"New cache not found\\") end"); then - echo "::warning::Could not verify the new build cache; keeping all previous caches." - exit 0 - fi - while read -r id; do - [ -n "$id" ] || continue - pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" - done <<< "$ids" - - # Compare what the pull request built with the released builds, so single packages - # can be bumped without a full rebuild; it reports and never fails the build. - - name: ABI check against the released builds - id: abi-check - if: inputs.platform == 'linux-64' - continue-on-error: true - run: | - pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR/$PLATFORM" --summary "abi-check-$DISTRO.md" - cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" - - name: Upload the ABI check - if: ${{ steps.abi-check.outcome == 'success' }} - uses: actions/upload-artifact@v6 - with: - name: abi-check-${{ inputs.distro }} - path: abi-check-${{ inputs.distro }}.md - retention-days: 7 - - # without recipes to build there is no recipes artifact, and nothing to stage - - name: Prepare the work directory - if: needs.plan.outputs.count != '0' - run: pixi run rs "$DISTRO" prepare - - if: needs.plan.outputs.count != '0' - uses: actions/download-artifact@v7 - with: - name: recipes-${{ inputs.distro }}-${{ inputs.platform }} - path: distros/${{ inputs.distro }}/work/recipes - - name: Generate the staged build workflows (catches post-merge problems) - if: needs.plan.outputs.count != '0' - run: | - for attempt in 1 2 3; do - if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi - if [[ "$attempt" == 3 ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi - echo "Workflow generation attempt $attempt failed; retrying..." >&2 - sleep 15 - done -""" - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/robostack.py b/tools/robostack.py index 41cdddbaa..b35d97113 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -386,7 +386,7 @@ def rattler_build(distro: str, args: list[str], skip_existing: bool = True) -> l # With uploads redirected (e.g. to a test channel), later stages of the staged # build need the packages earlier stages uploaded there, and reruns skip them. extra = upload_channel_url() - # the staged pull-request build's packages of earlier stages (tools/pr_stages.py) + # a staged build job's packages of the jobs it needs (tools/staged_build.py) local = os.environ.get("ROBOSTACK_LOCAL_CHANNEL", "").strip() cmd = ["rattler-build", "build", "-m", "./conda_build_config.yaml", *(["-c", (ROOT / local).resolve().as_uri()] if local else []), diff --git a/tools/staged_build.py b/tools/staged_build.py new file mode 100644 index 000000000..ef2b673c7 --- /dev/null +++ b/tools/staged_build.py @@ -0,0 +1,515 @@ +"""Staged builds: vinca-gha's stages as matrix jobs, for pull requests and releases. + + python tools/staged_build.py plan --distro --platform

--os --cache-dir

--out + python tools/staged_build.py schedule --out ... + python tools/staged_build.py job --schedule --distro --platform

--job + python tools/staged_build.py workflow [--check] + +.github/workflows/staged-build.yaml builds the given distributions and platforms in +stages, one job per batch of recipes. GitHub can't add jobs to a running workflow, so +it has a fixed number of stage slots (SLOTS) for all of them together: stage_ runs +stage k of every distribution and platform, each job as a matrix entry, after +stage_. + +- `plan` (one job per distribution and platform) lists the stages' jobs with + `vinca-gha --json`, leaving out the recipes whose every build is in the restored + pull-request cache. +- `schedule` puts the plans into the slots (at most MAX_MATRIX jobs each, GitHub's + matrix limit) and writes the schedule every stage job reads its recipes from. +- `job` is a stage job's recipes and the artifacts it needs: the packages of the jobs + that build its requirements (vinca's `needs`, and theirs), and the restored cache. +- `workflow` writes staged-build.yaml (`rs check` checks that it is up to date). + +Packages pass between jobs as artifacts, pkgs---, each with a +status/ file (success or failure): a job whose requirements failed fails too, but +the other distributions and platforms go on. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import build_timings +import robostack as rs + +SLOTS = 64 # lyrical's full linux-64 rebuild has 53 stages at --batch_size 25 +MAX_MATRIX = 256 # GitHub's limit of jobs per matrix +MAX_NAME = 60 # job names are a stage's packages, which can be many +WORKFLOW = rs.ROOT / ".github" / "workflows" / "staged-build.yaml" +PLACEHOLDER = {"distro": "-", "platform": "-", "os": "ubuntu-latest", "job": "-", "name": "-"} + + +def artifact(distro: str, platform: str, job: str) -> str: + return f"pkgs-{distro}-{platform}-{job}" + + +def cached_recipes(recipes: Path, platform: str, cache_dir: Path) -> set[str]: + """The recipes whose every build (one per variant) is in the cache (/*.conda).""" + names = {p.parent.name for p in recipes.glob("*/recipe.yaml")} + variants = build_timings.variant_builds(recipes, platform) if names else {} + cached = Counter(f.name.rsplit("-", 2)[0] for f in cache_dir.glob("*/*.conda")) + return {r for r in names if cached[r] >= variants.get(r, 1)} + + +def fold(stages: list[list[dict]], slots: int = SLOTS) -> list[list[dict]]: + """The stages as at most `slots` stages: the ones past the last slot become one job + there (build-ci orders a job's recipes by their dependencies).""" + if len(stages) <= slots: + return stages + rest = [job for stage in stages[slots - 1:] for job in stage] + ids = {job["id"] for job in rest} + tail = { + "id": f"stage_{slots - 1}_rest", + "name": f"{len(rest)} jobs of stages {slots - 1}+", + "recipes": " ".join(job["recipes"] for job in rest), + "needs": sorted({n for job in rest for n in job["needs"]} - ids), + } + return [*stages[:slots - 1], [tail]] + + +def plan(distro: str, platform: str, cache_dir: Path, batch_size: int) -> list[list[dict]]: + w = rs.work_dir(distro) + recipes = w / "recipes" + cached = cached_recipes(recipes, platform, cache_dir) if recipes.is_dir() else set() + # vinca-gha stages what is in ./recipes; the cached recipes count as built, like the + # ones on the channel that generate-recipes -n leaves out + aside = w / "recipes-cached" + shutil.rmtree(aside, ignore_errors=True) + aside.mkdir() + for r in cached: + (recipes / r).rename(aside / r) + try: + out = w / "stages.json" + cmd = ["vinca-gha", "--platform", platform, "-d", "./recipes", "--batch_size", str(batch_size), + "--json", out.name] + if build_timings.vinca_timings(distro, platform, w / "build_timings.json"): + cmd += ["--timings", "build_timings.json"] + if rs.run(cmd, w): + raise SystemExit("vinca-gha failed") + stages = json.loads(out.read_text())["stages"] + finally: + for r in cached: + (aside / r).rename(recipes / r) + aside.rmdir() + print(f"{len(cached)} recipes fully in the cache; {sum(map(len, stages))} jobs in {len(stages)} stages") + return fold(stages) + + +def merge_smallest(stages: list[list[dict]], k: int) -> None: + """Merge the two jobs of stage k with the fewest recipes into one.""" + a, b = sorted(stages[k], key=lambda job: len(job["recipes"].split()))[:2] + a["recipes"] = f"{a['recipes']} {b['recipes']}" + a["name"] = f"{a['name']} {b['name']}" + a["needs"] = sorted(set(a["needs"]) | set(b["needs"])) + stages[k].remove(b) + for stage in stages[k + 1:]: + for job in stage: + if b["id"] in job["needs"]: + job["needs"] = sorted((set(job["needs"]) - {b["id"]}) | {a["id"]}) + + +def schedule(plans: list[dict], slots: int = SLOTS, max_matrix: int = MAX_MATRIX) -> tuple[list[list[dict]], dict]: + """The slots' matrix entries, and each distribution and platform's jobs by id + ({"/": {id: {"recipes", "needs"}}}). A slot with more than + max_matrix jobs has the smallest jobs of its largest plans merged.""" + entries = [] + for k in range(slots): + while sum(len(p["stages"][k]) for p in plans if k < len(p["stages"])) > max_matrix: + largest = max((p for p in plans if k < len(p["stages"])), key=lambda p: len(p["stages"][k])) + merge_smallest(largest["stages"], k) + entries.append([ + {"distro": p["distro"], "platform": p["platform"], "os": p["os"], "job": job["id"], + "name": short(f"{p['distro']} {p['platform']} {job['name']}")} + for p in plans if k < len(p["stages"]) for job in p["stages"][k] + ]) + while entries and not entries[-1]: + entries.pop() + jobs = {f"{p['distro']}/{p['platform']}": {job["id"]: {"recipes": job["recipes"], "needs": job["needs"]} + for stage in p["stages"] for job in stage} + for p in plans} + return entries, jobs + + +def short(name: str) -> str: + return name if len(name) <= MAX_NAME else name[:MAX_NAME - 1] + "…" + + +def job_needs(jobs: dict[str, dict], job: str) -> list[str]: + """The jobs whose packages the job needs: its needs, and theirs.""" + seen: set[str] = set() + todo = list(jobs[job]["needs"]) + while todo: + if (j := todo.pop()) not in seen: + seen.add(j) + todo += jobs[j]["needs"] + return sorted(seen) + + +def pattern(distro: str, platform: str, jobs: list[str]) -> str: + """download-artifact's pattern (minimatch) for these jobs' artifacts.""" + names = ["cache", *jobs] + return artifact(distro, platform, names[0] if len(names) == 1 else "{" + ",".join(names) + "}") + + +def workflow() -> str: + """staged-build.yaml: HEADER, the stage slots, then COLLECT.""" + secrets = json.dumps({d: rs.anaconda_token_secret(d) for d in rs.distros()}, separators=(",", ":")) + slots = [] + for k in range(SLOTS): + needs = "schedule" if k == 0 else f"[schedule, stage_{k - 1}]" + slots.append(SLOT.replace("@K@", str(k)).replace("@NEEDS@", needs).replace("@SECRETS@", secrets)) + outputs = "".join(f" slot_{k}: ${{{{ steps.schedule.outputs.slot_{k} }}}}\n" for k in range(SLOTS)) + every = ", ".join(["schedule", *(f"stage_{k}" for k in range(SLOTS))]) + return HEADER.replace("@OUTPUTS@", outputs) + "".join(slots) + COLLECT.replace("@NEEDS@", every) + + +def write_lines(env: str, lines: list[str]) -> None: + if path := os.environ.get(env): + with open(path, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("plan", help="the stages of one distribution and platform") + p.add_argument("--distro", required=True) + p.add_argument("--platform", required=True) + p.add_argument("--os", required=True, help="the platform's runner") + p.add_argument("--cache-dir", required=True, type=Path) + p.add_argument("--batch-size", type=int, default=25) + p.add_argument("--out", required=True, type=Path) + s = sub.add_parser("schedule", help="the plans in the stage slots, as GitHub outputs") + s.add_argument("--out", required=True, type=Path) + s.add_argument("plans", nargs="*", type=Path) + j = sub.add_parser("job", help="a stage job's RECIPES, NEEDED and NEEDS_PATTERN, as GitHub environment") + j.add_argument("--schedule", required=True, type=Path) + j.add_argument("--distro", required=True) + j.add_argument("--platform", required=True) + j.add_argument("--job", required=True) + w = sub.add_parser("workflow", help=f"write {WORKFLOW.relative_to(rs.ROOT)}") + w.add_argument("--check", action="store_true", help="only check that it is up to date") + args = parser.parse_args() + + if args.command == "workflow": + text = workflow() + if args.check: + if not WORKFLOW.is_file() or WORKFLOW.read_text() != text: + print(f"{WORKFLOW.relative_to(rs.ROOT)} is out of date (python tools/staged_build.py workflow)") + return 1 + return 0 + WORKFLOW.write_text(text) + return 0 + + if args.command == "plan": + stages = plan(args.distro, args.platform, args.cache_dir.resolve(), args.batch_size) + doc = {"distro": args.distro, "platform": args.platform, "os": args.os, "stages": stages} + args.out.write_text(json.dumps(doc) + "\n") + jobs = sum(map(len, stages)) + write_lines("GITHUB_STEP_SUMMARY", [f"**{args.distro} {args.platform}**: {jobs} build jobs in {len(stages)} stages", ""]) + for k, stage in enumerate(stages): + print(f"stage {k}: " + "; ".join(job["recipes"] for job in stage)) + return 0 + + if args.command == "schedule": + plans = [json.loads(f.read_text()) for f in sorted(args.plans)] + entries, jobs = schedule(plans) + args.out.write_text(json.dumps(jobs) + "\n") + # the slots past the last stage are skipped, but GitHub reads their matrix anyway + lines = [f"count={len(entries)}", + "combos=" + json.dumps([{"distro": p["distro"], "platform": p["platform"]} for p in plans])] + lines += [f"slot_{k}=" + json.dumps(entries[k] if k < len(entries) else [PLACEHOLDER], separators=(",", ":")) + for k in range(SLOTS)] + write_lines("GITHUB_OUTPUT", lines) + print(f"{sum(map(len, entries))} jobs in {len(entries)} stages: " + ", ".join(str(len(e)) for e in entries)) + return 0 + + jobs = json.loads(args.schedule.read_text())[f"{args.distro}/{args.platform}"] + needed = job_needs(jobs, args.job) + lines = [f"RECIPES={jobs[args.job]['recipes']}", f"NEEDED={' '.join(needed)}", + f"NEEDS_PATTERN={pattern(args.distro, args.platform, needed)}"] + write_lines("GITHUB_ENV", lines) + print("\n".join(lines)) + return 0 + + +SETUP = """\ + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) +""" + +HEADER = """\ +# Generated by tools/staged_build.py (python tools/staged_build.py workflow); `pixi run rs check` +# checks that it is up to date. +# +# The given distributions and platforms, built in vinca-gha's stages, one job per batch +# of recipes (stage-job.yaml): pull requests (testpr.yaml) keep the packages as the +# pull request's build cache, releases (release.yaml) upload them. See +# tools/staged_build.py for how the stages become jobs. +name: Staged build + +on: + workflow_call: + inputs: + distros: + description: JSON list of distributions + required: true + type: string + targets: + description: 'JSON list of platforms and their runners, [{"os": ..., "platform": ...}]' + required: true + type: string + upload: + description: Upload the packages to the distributions' channels (else keep them as the pull request's cache) + required: true + type: boolean + +env: + PYTHONIOENCODING: utf-8 + # while uploads go to a test channel, packages already there aren't built again + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} + # the pull request's packages, /*.conda; restored and saved on Linux for every platform + CACHE_DIR: pr-cache + +jobs: + plan: + strategy: + fail-fast: false + matrix: + distro: ${{ fromJSON(inputs.distros) }} + target: ${{ fromJSON(inputs.targets) }} + name: plan ${{ matrix.distro }} ${{ matrix.target.platform }} + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.target.platform }} + steps: +""" + SETUP + """\ + - name: Generate recipes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi + echo "Recipe generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done + - name: Upload the recipes + uses: actions/upload-artifact@v6 + with: + name: recipes-${{ matrix.distro }}-${{ matrix.target.platform }} + path: distros/${{ matrix.distro }}/work/recipes + if-no-files-found: ignore + overwrite: true + retention-days: 7 + - name: Restore build cache + if: ${{ !inputs.upload }} + uses: actions/cache/restore@v5 + with: + path: ${{ env.CACHE_DIR }} + key: staged-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }} + restore-keys: | + staged-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}- + # Temporary rebuild controls live in distros//ci.yaml. + - name: Apply ci.yaml cache controls + run: | + for subdir in "$PLATFORM" noarch; do + mkdir -p "$CACHE_DIR/$subdir" + pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR/$subdir" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" + done + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages restored from the cache" + for package in "${packages[@]}"; do basename "$package"; done + mkdir -p "$CACHE_DIR/status" && echo success > "$CACHE_DIR/status/cache" + - name: Upload the cached packages + uses: actions/upload-artifact@v6 + with: + name: pkgs-${{ matrix.distro }}-${{ matrix.target.platform }}-cache + path: ${{ env.CACHE_DIR }} + compression-level: 0 # .conda files are compressed + overwrite: true + retention-days: 3 + - name: Estimate the build time + run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR/$PLATFORM" --platform "$PLATFORM" + - name: Plan the stages + env: + OS: ${{ matrix.target.os }} + run: pixi run python tools/staged_build.py plan --distro "$DISTRO" --platform "$PLATFORM" --os "$OS" --cache-dir "$CACHE_DIR" --out "plan-$DISTRO-$PLATFORM.json" + - uses: actions/upload-artifact@v6 + with: + name: plan-${{ matrix.distro }}-${{ matrix.target.platform }} + path: plan-${{ matrix.distro }}-${{ matrix.target.platform }}.json + overwrite: true + retention-days: 7 + + # Every plan in the stage slots, and the schedule the stage jobs read their recipes from; + # a failed plan leaves out only its distribution and platform. + schedule: + needs: plan + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + outputs: + count: ${{ steps.schedule.outputs.count }} + combos: ${{ steps.schedule.outputs.combos }} +@OUTPUTS@ steps: +""" + SETUP + """\ + - uses: actions/download-artifact@v7 + with: + pattern: plan-* + merge-multiple: true + path: plans + - name: Schedule the stages + id: schedule + run: pixi run python tools/staged_build.py schedule --out schedule.json plans/*.json + - uses: actions/upload-artifact@v6 + with: + name: schedule + path: schedule.json + overwrite: true + retention-days: 7 + +""" + +# A slot runs after the one before even when that one failed: the jobs check that +# their own requirements built (stage-job.yaml), so the other distributions and +# platforms go on. +SLOT = """\ + stage_@K@: + needs: @NEEDS@ + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > @K@ }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_@K@) }} + name: @K@ ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('@SECRETS@')[matrix.distro] }} + +""" + +COLLECT = """\ + # Pull requests: the packages of every stage as the pull request's build cache (a new + # push builds only what changed), then the ABI check of what the pull request built. + collect: + needs: [@NEEDS@] + if: ${{ always() && !inputs.upload && needs.schedule.result == 'success' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.combos) }} + name: collect ${{ matrix.distro }} ${{ matrix.platform }} + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.platform }} + steps: +""" + SETUP + """\ + - uses: actions/download-artifact@v7 + with: + pattern: pkgs-${{ matrix.distro }}-${{ matrix.platform }}-* + merge-multiple: true + path: ${{ env.CACHE_DIR }} + - name: See the packages + run: | + rm -rf "$CACHE_DIR/status" + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages" + for package in "${packages[@]}"; do basename "$package"; done + - name: Save build cache + id: save-cache + uses: actions/cache/save@v5 + with: + path: ${{ env.CACHE_DIR }} + key: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + + # Only the newest cache of a pull request/distribution/platform is restored; the ones + # of earlier runs and attempts only push others out of the repository's 10 GB, so + # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. + - name: Drop this pull request's superseded build caches + if: ${{ steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PREFIX: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}- + KEEP: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + run: | + # cache/save reports success even on upload failure: verify the replacement exists. + if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \\ + --jq "if any(.[]; .key == \\"$KEEP\\") then .[] | select(.key != \\"$KEEP\\") | .id else error(\\"New cache not found\\") end"); then + echo "::warning::Could not verify the new build cache; keeping all previous caches." + exit 0 + fi + while read -r id; do + [ -n "$id" ] || continue + pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" + done <<< "$ids" + + # Compare what the pull request built with the released builds, so single packages + # can be bumped without a full rebuild; it reports and never fails the build. + - name: ABI check against the released builds + id: abi-check + if: matrix.platform == 'linux-64' + continue-on-error: true + run: | + pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR/$PLATFORM" --summary "abi-check-$DISTRO.md" + cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" + - name: Upload the ABI check + if: ${{ steps.abi-check.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: abi-check-${{ matrix.distro }} + path: abi-check-${{ matrix.distro }}.md + retention-days: 7 + + # the build branches' workflows, while they still build the releases; without + # recipes to build there is no recipes artifact, and nothing to stage + - name: Prepare the work directory + run: pixi run rs "$DISTRO" prepare + # a pattern, which (unlike a name) may match nothing + - uses: actions/download-artifact@v7 + with: + pattern: recipes-${{ matrix.distro }}-${{ matrix.platform }} + merge-multiple: true + path: distros/${{ matrix.distro }}/work/recipes + - name: Generate the staged build workflows (catches post-merge problems) + run: | + if [ -z "$(ls "distros/$DISTRO/work/recipes" 2>/dev/null)" ]; then echo "Nothing to build"; exit 0; fi + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi + echo "Workflow generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done +""" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/test_ci_build.py b/tools/tests/test_ci_build.py index b06c2af02..1b3acc8ac 100644 --- a/tools/tests/test_ci_build.py +++ b/tools/tests/test_ci_build.py @@ -80,7 +80,7 @@ def test_interruption_stops_descendants_and_preserves_only_finished_archives(tmp @pytest.mark.parametrize("lookup_status,ids,deleted", [(0, "11\n22", "11\n22\n"), (1, "11", ""), (0, "", "")]) def test_cache_pruning_requires_verified_replacement(tmp_path, lookup_status, ids, deleted): - workflow = yaml.safe_load((ROOT / ".github/workflows/pr-stages.yaml").read_text()) + workflow = yaml.safe_load((ROOT / ".github/workflows/staged-build.yaml").read_text()) prune = next(s for s in workflow["jobs"]["collect"]["steps"] if s.get("name") == "Drop this pull request's superseded build caches") pixi = tmp_path / "pixi" diff --git a/tools/tests/test_pr_stages.py b/tools/tests/test_pr_stages.py deleted file mode 100644 index 177788b54..000000000 --- a/tools/tests/test_pr_stages.py +++ /dev/null @@ -1,78 +0,0 @@ -"""tools/pr_stages.py: the staged pull-request build's plan and stage slots.""" - -import yaml - -import pr_stages -import robostack as rs - - -def job(recipes: str) -> dict: - return {"name": recipes, "recipes": recipes} - - -def test_fit_numbers_the_jobs_for_their_artifacts(): - assert pr_stages.fit([[job("ros2-a")], [job("ros2-b"), job("ros2-c")]]) == [ - [{"id": "s0-0", "name": "ros2-a", "recipes": "ros2-a"}], - [{"id": "s1-0", "name": "ros2-b", "recipes": "ros2-b"}, {"id": "s1-1", "name": "ros2-c", "recipes": "ros2-c"}], - ] - - -def test_fit_puts_the_stages_past_the_last_slot_into_one_job(): - stages = [[job("ros2-a")], [job("ros2-b")], [job("ros2-c"), job("ros2-d")], [job("ros2-e")]] - - fitted = pr_stages.fit(stages, slots=3) - - assert len(fitted) == 3 - assert [j["recipes"] for j in fitted[2]] == ["ros2-c ros2-d ros2-e"] - - -def test_fit_shortens_long_job_names(): - (fitted,) = pr_stages.fit([[{"name": "x" * 100, "recipes": "ros2-x"}]]) - assert len(fitted[0]["name"]) == pr_stages.MAX_NAME - - -def test_cached_recipes_need_every_variant_in_the_cache(monkeypatch, tmp_path): - recipes, cache = tmp_path / "recipes", tmp_path / "cache" - for name in ("ros2-a", "ros2-b", "ros2-c"): - (recipes / name).mkdir(parents=True) - (recipes / name / "recipe.yaml").write_text("{}\n") - (cache / "linux-64").mkdir(parents=True) - (cache / "noarch").mkdir() - (cache / "linux-64" / "ros2-a-1.0.0-py312_0.conda").touch() - (cache / "linux-64" / "ros2-a-1.0.0-py313_0.conda").touch() - (cache / "linux-64" / "ros2-b-1.0.0-py312_0.conda").touch() - (cache / "noarch" / "ros2-c-1.0.0-0.conda").touch() - monkeypatch.setattr(pr_stages.build_timings, "variant_builds", - lambda path, platform: {"ros2-a": 2, "ros2-b": 2}) - - assert pr_stages.cached_recipes(recipes, "linux-64", cache) == {"ros2-a", "ros2-c"} - - -def test_every_slot_waits_for_the_one_before(): - jobs = yaml.safe_load(pr_stages.workflow())["jobs"] - - assert jobs["stage_0"]["needs"] == "plan" - for k in range(1, pr_stages.SLOTS): - assert jobs[f"stage_{k}"]["needs"] == ["plan", f"stage_{k - 1}"] - assert f"> {k} }}" in jobs[f"stage_{k}"]["if"] - assert jobs["collect"]["needs"] == ["plan", *(f"stage_{k}" for k in range(pr_stages.SLOTS))] - - -def test_the_workflow_is_up_to_date(): - assert pr_stages.WORKFLOW.read_text() == pr_stages.workflow(), "python tools/pr_stages.py workflow" - - -def test_rattler_build_reads_the_local_channel(monkeypatch): - monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False) - monkeypatch.setenv("ROBOSTACK_LOCAL_CHANNEL", "pr-channel") - - cmd = rs.rattler_build("jazzy", []) - - assert cmd[cmd.index("-c") + 1] == (rs.ROOT / "pr-channel").as_uri() - - -def test_rattler_build_without_the_local_channel(monkeypatch): - monkeypatch.delenv("ROBOSTACK_LOCAL_CHANNEL", raising=False) - monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False) - - assert not any(c.startswith("file:") for c in rs.rattler_build("jazzy", [])) diff --git a/tools/tests/test_staged_build.py b/tools/tests/test_staged_build.py new file mode 100644 index 000000000..916060022 --- /dev/null +++ b/tools/tests/test_staged_build.py @@ -0,0 +1,105 @@ +"""tools/staged_build.py: the staged build's plans, schedule and stage slots.""" + +import yaml + +import robostack as rs +import staged_build as sb + + +def job(id: str, recipes: str, needs=()) -> dict: + return {"id": id, "name": recipes, "recipes": recipes, "needs": list(needs)} + + +def plan(distro: str, platform: str, stages: list[list[dict]]) -> dict: + return {"distro": distro, "platform": platform, "os": "ubuntu-latest", "stages": stages} + + +def test_fold_puts_the_stages_past_the_last_slot_into_one_job(): + stages = [[job("s0", "ros2-a")], [job("s1", "ros2-b", ["s0"])], + [job("s2a", "ros2-c", ["s1"]), job("s2b", "ros2-d", ["s0"])], [job("s3", "ros2-e", ["s2a"])]] + + folded = sb.fold(stages, slots=3) + + assert len(folded) == 3 + (tail,) = folded[2] + assert tail["recipes"] == "ros2-c ros2-d ros2-e" + assert tail["needs"] == ["s0", "s1"] + + +def test_cached_recipes_need_every_variant_in_the_cache(monkeypatch, tmp_path): + recipes, cache = tmp_path / "recipes", tmp_path / "cache" + for name in ("ros2-a", "ros2-b", "ros2-c"): + (recipes / name).mkdir(parents=True) + (recipes / name / "recipe.yaml").write_text("{}\n") + (cache / "linux-64").mkdir(parents=True) + (cache / "noarch").mkdir() + (cache / "linux-64" / "ros2-a-1.0.0-py312_0.conda").touch() + (cache / "linux-64" / "ros2-a-1.0.0-py313_0.conda").touch() + (cache / "linux-64" / "ros2-b-1.0.0-py312_0.conda").touch() + (cache / "noarch" / "ros2-c-1.0.0-0.conda").touch() + monkeypatch.setattr(sb.build_timings, "variant_builds", lambda path, platform: {"ros2-a": 2, "ros2-b": 2}) + + assert sb.cached_recipes(recipes, "linux-64", cache) == {"ros2-a", "ros2-c"} + + +def test_schedule_shares_the_slots_between_distributions_and_platforms(): + plans = [plan("jazzy", "linux-64", [[job("stage_0_job_0", "ros2-a")], [job("stage_1_job_0", "ros2-b")]]), + plan("lyrical", "win-64", [[job("stage_0_job_0", "ros2-a"), job("stage_0_job_1", "ros2-c")]])] + + entries, jobs = sb.schedule(plans) + + assert [[(e["distro"], e["platform"], e["job"]) for e in slot] for slot in entries] == [ + [("jazzy", "linux-64", "stage_0_job_0"), ("lyrical", "win-64", "stage_0_job_0"), + ("lyrical", "win-64", "stage_0_job_1")], + [("jazzy", "linux-64", "stage_1_job_0")], + ] + assert jobs["lyrical/win-64"]["stage_0_job_1"] == {"recipes": "ros2-c", "needs": []} + + +def test_schedule_merges_jobs_past_the_matrix_limit(): + stages = [[job("a", "ros2-a"), job("b", "ros2-b ros2-bb"), job("c", "ros2-c")], [job("d", "ros2-d", ["c"])]] + + entries, jobs = sb.schedule([plan("jazzy", "linux-64", stages)], max_matrix=2) + + assert [e["job"] for e in entries[0]] == ["a", "b"] + assert jobs["jazzy/linux-64"]["a"]["recipes"] == "ros2-a ros2-c" + assert jobs["jazzy/linux-64"]["d"]["needs"] == ["a"] + + +def test_a_job_needs_its_requirements_jobs_and_theirs(): + jobs = {"a": {"needs": []}, "b": {"needs": ["a"]}, "c": {"needs": []}, "d": {"needs": ["b"]}} + + assert sb.job_needs(jobs, "d") == ["a", "b"] + assert sb.pattern("jazzy", "linux-64", ["a", "b"]) == "pkgs-jazzy-linux-64-{cache,a,b}" + assert sb.pattern("jazzy", "linux-64", []) == "pkgs-jazzy-linux-64-cache" + + +def test_every_slot_runs_after_the_one_before(): + jobs = yaml.safe_load(sb.workflow())["jobs"] + + assert jobs["stage_0"]["needs"] == "schedule" + for k in range(1, sb.SLOTS): + assert jobs[f"stage_{k}"]["needs"] == ["schedule", f"stage_{k - 1}"] + assert f"> {k} }}" in jobs[f"stage_{k}"]["if"] + assert jobs["collect"]["needs"] == ["schedule", *(f"stage_{k}" for k in range(sb.SLOTS))] + assert set(jobs["schedule"]["outputs"]) == {"count", "combos", *(f"slot_{k}" for k in range(sb.SLOTS))} + + +def test_the_workflow_is_up_to_date(): + assert sb.WORKFLOW.read_text() == sb.workflow(), "python tools/staged_build.py workflow" + + +def test_rattler_build_reads_the_local_channel(monkeypatch): + monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False) + monkeypatch.setenv("ROBOSTACK_LOCAL_CHANNEL", "pr-channel") + + cmd = rs.rattler_build("jazzy", []) + + assert cmd[cmd.index("-c") + 1] == (rs.ROOT / "pr-channel").as_uri() + + +def test_rattler_build_without_the_local_channel(monkeypatch): + monkeypatch.delenv("ROBOSTACK_LOCAL_CHANNEL", raising=False) + monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False) + + assert not any(c.startswith("file:") for c in rs.rattler_build("jazzy", [])) From f138be6bd977bfe08aa04e530ff52a5c62010863 Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Thu, 12 Nov 2026 17:57:37 -0700 Subject: [PATCH 3/6] DO NOT MERGE: try the staged build with a lyrical rebuild on linux-64 and win-64 --- .github/workflows/testpr.yaml | 3 +++ distros/lyrical/vinca.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index bcaa57afc..3cab799ac 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -65,6 +65,9 @@ jobs: echo "distros=$distros" | tee -a "$GITHUB_OUTPUT" # build:linux / build:osx / build:win labels limit the platforms (all without) echo "targets=$(pixi run python tools/robostack.py build-targets --labels "$labels")" | tee -a "$GITHUB_OUTPUT" + # DO NOT MERGE: only lyrical on linux-64 and win-64, to try the staged build + echo 'distros=["lyrical"]' | tee -a "$GITHUB_OUTPUT" + echo 'targets=[{"os": "ubuntu-latest", "platform": "linux-64"}, {"os": "windows-2022", "platform": "win-64"}]' | tee -a "$GITHUB_OUTPUT" check: runs-on: ubuntu-latest diff --git a/distros/lyrical/vinca.yaml b/distros/lyrical/vinca.yaml index 15070a041..a5ec3054a 100644 --- a/distros/lyrical/vinca.yaml +++ b/distros/lyrical/vinca.yaml @@ -3,7 +3,7 @@ ros_distro: lyrical patch_dir: patch rosdistro_snapshot: rosdistro_snapshot.yaml rosdistro_additional_recipes: rosdistro_additional_recipes.yaml -build_number: 30 +build_number: 31 mutex_package: version: 0.22.0 From 0126c36c08b3633096987f3703bc10a19748e6d4 Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Thu, 12 Nov 2026 18:06:24 -0700 Subject: [PATCH 4/6] Pass the recipes as a tarball: artifacts drop the build scripts' executable bit --- .github/workflows/stage-job.yaml | 9 ++++----- .github/workflows/staged-build.yaml | 18 ++++++++---------- tools/staged_build.py | 18 ++++++++---------- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/.github/workflows/stage-job.yaml b/.github/workflows/stage-job.yaml index 230073800..ef5584869 100644 --- a/.github/workflows/stage-job.yaml +++ b/.github/workflows/stage-job.yaml @@ -122,16 +122,15 @@ jobs: shell: bash -l {0} run: pixi run python tools/staged_build.py job --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" --job "$JOB" - - name: Prepare the work directory - if: steps.earlier.outputs.built != 'true' - shell: bash -l {0} - run: pixi run rs "$DISTRO" prepare # the plan job's recipes (generated once for all the jobs) - uses: actions/download-artifact@v7 if: steps.earlier.outputs.built != 'true' with: name: recipes-${{ inputs.distro }}-${{ inputs.platform }} - path: distros/${{ inputs.distro }}/work/recipes + - name: Prepare the work directory + if: steps.earlier.outputs.built != 'true' + shell: bash -l {0} + run: pixi run rs "$DISTRO" prepare && tar -xf recipes.tar -C "distros/$DISTRO/work" # the packages of the jobs this one needs, and the restored cache - uses: actions/download-artifact@v7 if: steps.earlier.outputs.built != 'true' diff --git a/.github/workflows/staged-build.yaml b/.github/workflows/staged-build.yaml index 16f7e5aac..c7f254620 100644 --- a/.github/workflows/staged-build.yaml +++ b/.github/workflows/staged-build.yaml @@ -66,12 +66,14 @@ jobs: echo "Recipe generation attempt $attempt failed; retrying..." >&2 sleep 15 done + # a tarball: artifacts drop the build scripts' executable bit + - name: Pack the recipes + run: mkdir -p "distros/$DISTRO/work/recipes" && tar -cf recipes.tar -C "distros/$DISTRO/work" recipes - name: Upload the recipes uses: actions/upload-artifact@v6 with: name: recipes-${{ matrix.distro }}-${{ matrix.target.platform }} - path: distros/${{ matrix.distro }}/work/recipes - if-no-files-found: ignore + path: recipes.tar overwrite: true retention-days: 7 - name: Restore build cache @@ -1453,16 +1455,12 @@ jobs: path: abi-check-${{ matrix.distro }}.md retention-days: 7 - # the build branches' workflows, while they still build the releases; without - # recipes to build there is no recipes artifact, and nothing to stage - - name: Prepare the work directory - run: pixi run rs "$DISTRO" prepare - # a pattern, which (unlike a name) may match nothing + # the build branches' workflows, while they still build the releases - uses: actions/download-artifact@v7 with: - pattern: recipes-${{ matrix.distro }}-${{ matrix.platform }} - merge-multiple: true - path: distros/${{ matrix.distro }}/work/recipes + name: recipes-${{ matrix.distro }}-${{ matrix.platform }} + - name: Prepare the work directory + run: pixi run rs "$DISTRO" prepare && tar -xf recipes.tar -C "distros/$DISTRO/work" - name: Generate the staged build workflows (catches post-merge problems) run: | if [ -z "$(ls "distros/$DISTRO/work/recipes" 2>/dev/null)" ]; then echo "Nothing to build"; exit 0; fi diff --git a/tools/staged_build.py b/tools/staged_build.py index ef2b673c7..75d6b559f 100644 --- a/tools/staged_build.py +++ b/tools/staged_build.py @@ -313,12 +313,14 @@ def main() -> int: echo "Recipe generation attempt $attempt failed; retrying..." >&2 sleep 15 done + # a tarball: artifacts drop the build scripts' executable bit + - name: Pack the recipes + run: mkdir -p "distros/$DISTRO/work/recipes" && tar -cf recipes.tar -C "distros/$DISTRO/work" recipes - name: Upload the recipes uses: actions/upload-artifact@v6 with: name: recipes-${{ matrix.distro }}-${{ matrix.target.platform }} - path: distros/${{ matrix.distro }}/work/recipes - if-no-files-found: ignore + path: recipes.tar overwrite: true retention-days: 7 - name: Restore build cache @@ -489,16 +491,12 @@ def main() -> int: path: abi-check-${{ matrix.distro }}.md retention-days: 7 - # the build branches' workflows, while they still build the releases; without - # recipes to build there is no recipes artifact, and nothing to stage - - name: Prepare the work directory - run: pixi run rs "$DISTRO" prepare - # a pattern, which (unlike a name) may match nothing + # the build branches' workflows, while they still build the releases - uses: actions/download-artifact@v7 with: - pattern: recipes-${{ matrix.distro }}-${{ matrix.platform }} - merge-multiple: true - path: distros/${{ matrix.distro }}/work/recipes + name: recipes-${{ matrix.distro }}-${{ matrix.platform }} + - name: Prepare the work directory + run: pixi run rs "$DISTRO" prepare && tar -xf recipes.tar -C "distros/$DISTRO/work" - name: Generate the staged build workflows (catches post-merge problems) run: | if [ -z "$(ls "distros/$DISTRO/work/recipes" 2>/dev/null)" ]; then echo "Nothing to build"; exit 0; fi From 3bd34fa3261592b9b7df0ded7e8e5e1aa695ee7e Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Fri, 13 Nov 2026 08:30:11 -0700 Subject: [PATCH 5/6] Staged builds: download artifacts by name, and a status per recipe download-artifact's pattern only sees a run's first 1,000 artifacts: the stage jobs and collect download the artifacts they need by name (tools/download_artifacts.cjs), and the build log is an artifact only after a failure. A failed recipe now holds back only the recipes that need it (vinca-gha's requires): the job builds the rest with --continue-on-failure=skip-dependents, and status/.json records each recipe's outcome for the later jobs. Releases upload the packages of the recipes that built. --- .github/workflows/stage-job.yaml | 80 ++++++------- .github/workflows/staged-build.yaml | 13 +- pixi.lock | 12 +- pixi.toml | 2 +- tools/download_artifacts.cjs | 104 ++++++++++++++++ tools/robostack.py | 15 ++- tools/staged_build.py | 176 ++++++++++++++++++++++++---- tools/tests/test_staged_build.py | 83 ++++++++++++- 8 files changed, 405 insertions(+), 80 deletions(-) create mode 100644 tools/download_artifacts.cjs diff --git a/.github/workflows/stage-job.yaml b/.github/workflows/stage-job.yaml index ef5584869..406d9aaa2 100644 --- a/.github/workflows/stage-job.yaml +++ b/.github/workflows/stage-job.yaml @@ -1,7 +1,8 @@ # One build job of a staged build (staged-build.yaml, tools/staged_build.py): builds its -# recipes with the packages of the jobs it needs (and the restored cache) as a local -# channel, and passes what it built on as an artifact, pkgs---, -# with status/; releases also upload it to the distribution's channels. +# recipes whose requirements were built, with the packages of the jobs it needs (and the +# restored cache) as a local channel, and passes what it built on as an artifact, +# pkgs---, with status/ and status/.json (each recipe's); +# releases also upload what built to the distribution's channels. name: Stage job on: @@ -45,28 +46,29 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 350 steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # Re-running failed jobs re-runs every later stage too: the jobs that succeeded # before keep their packages. - - uses: actions/download-artifact@v7 + - name: Download this job's packages of an earlier attempt + uses: actions/github-script@v8 with: - pattern: ${{ env.ARTIFACT }} - merge-multiple: true - path: earlier + script: | + await require('./tools/download_artifacts.cjs')({ names: [process.env.ARTIFACT], dir: 'earlier', core }) - name: Built in an earlier attempt? id: earlier shell: bash run: | - if [ "$(cat "earlier/status/$JOB" 2>/dev/null)" == success ]; then + # (python3 on Windows can be the Microsoft Store's placeholder) + status=$("$(command -v python || command -v python3)" -c 'import sys, zipfile; print(zipfile.ZipFile(sys.argv[1]).read(sys.argv[2]).decode().strip())' \ + "earlier/$ARTIFACT.zip" "status/$JOB" 2>/dev/null || true) + if [ "$status" == success ]; then echo "Built in an earlier attempt of this run" echo "built=true" >> "$GITHUB_OUTPUT" fi - # (clears the directory, earlier/ included: it isn't a repository yet) - - uses: actions/checkout@v6 - if: steps.earlier.outputs.built != 'true' - with: - persist-credentials: false - # git auto-maintenance can leave a lock file in the git clones pixi caches # (e.g. vinca's), which breaks every later run restoring that cache - name: Disable git auto-maintenance @@ -131,25 +133,20 @@ jobs: if: steps.earlier.outputs.built != 'true' shell: bash -l {0} run: pixi run rs "$DISTRO" prepare && tar -xf recipes.tar -C "distros/$DISTRO/work" - # the packages of the jobs this one needs, and the restored cache - - uses: actions/download-artifact@v7 + # by name: download-artifact's pattern only sees the run's first 1,000 artifacts + - name: Download the packages of the jobs this one needs, and the restored cache if: steps.earlier.outputs.built != 'true' + uses: actions/github-script@v8 with: - pattern: ${{ env.NEEDS_PATTERN }} - merge-multiple: true - path: pr-channel - - name: Check and index the packages of the jobs this one needs + script: | + await require('./tools/download_artifacts.cjs')({ names: process.env.NEEDED_ARTIFACTS.split(' '), dir: 'needed', core }) + # the recipes that need one that wasn't built are left out (and recorded as blocked) + - name: The recipes whose requirements were built if: steps.earlier.outputs.built != 'true' shell: bash -l {0} run: | - failed=() - for job in $NEEDED; do - [ "$(cat "pr-channel/status/$job" 2>/dev/null)" == success ] || failed+=("$job") - done - if (( ${#failed[@]} )); then - echo "::error::Not built, which this job needs: ${failed[*]} (see their jobs)" - exit 1 - fi + pixi run python tools/staged_build.py unpack --zips needed --into pr-channel + pixi run python tools/staged_build.py ready --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" --job "$JOB" --channel pr-channel rm -rf pr-channel/status mkdir -p "pr-channel/$PLATFORM" pr-channel/noarch shopt -s nullglob @@ -159,7 +156,7 @@ jobs: - name: Build recipes id: build-recipes - if: steps.earlier.outputs.built != 'true' + if: steps.earlier.outputs.built != 'true' && env.RECIPES != '' shell: bash -l {0} run: | set +e @@ -167,8 +164,9 @@ jobs: for recipe in $RECIPES; do args+=(--recipe "./recipes/$recipe"); done # shorter work dirs for MAX_PATH on Windows [ "$PLATFORM" == "win-64" ] && args+=(--no-build-id) - # the cache's packages are in pr-channel; build-ci builds independent recipes side by side - pixi run rs "$DISTRO" build-ci "${args[@]}" --skip-existing=all 2>&1 | tee rattler-build.log + # the cache's packages are in pr-channel; build-ci builds independent recipes side by + # side, and after a failed one goes on with those that don't need it + pixi run rs "$DISTRO" build-ci "${args[@]}" --skip-existing=all --continue-on-failure=skip-dependents 2>&1 | tee rattler-build.log exit "${PIPESTATUS[0]}" - name: Summarize build result @@ -179,12 +177,10 @@ jobs: --platform "$DISTRO $PLATFORM" --outcome "${{ steps.build-recipes.outcome }}" >> "$GITHUB_STEP_SUMMARY" # also after a failure: what was built goes into the pull request's cache, and the - # status tells the later jobs + # status tells the later jobs which recipes they can build - name: Collect the packages if: ${{ (success() || failure()) && steps.earlier.outputs.built != 'true' }} shell: bash -l {0} - env: - OUTCOME: ${{ steps.build-recipes.outcome }} run: | shopt -s nullglob mkdir -p pkgs/status @@ -192,8 +188,9 @@ jobs: files=("$OUTPUT_DIR/$subdir"/*.conda) if (( ${#files[@]} )); then mkdir -p "pkgs/$subdir" && cp "${files[@]}" "pkgs/$subdir/"; fi done - echo "$OUTCOME" > "pkgs/status/$JOB" (cd pkgs && printf '%s\n' ./*/*.conda) + pixi run python tools/staged_build.py status --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" --job "$JOB" \ + --log rattler-build.log --out pkgs - name: Upload the packages if: ${{ (success() || failure()) && steps.earlier.outputs.built != 'true' }} uses: actions/upload-artifact@v6 @@ -204,22 +201,25 @@ jobs: overwrite: true # re-runs of a failed job retention-days: 3 + # The packages of the recipes that built (uploads.txt, from the status step). # prefix.dev: trusted publishing (the caller grants id-token: write); anaconda.org: # the distribution's token. Both skip packages that are already there. - name: Upload to the distribution's channels - if: ${{ inputs.upload && steps.build-recipes.outcome == 'success' }} + if: ${{ inputs.upload && (success() || failure()) && steps.build-recipes.outcome != 'skipped' }} shell: bash -l {0} env: ANACONDA_API_TOKEN: ${{ secrets[inputs.token_secret] }} run: | - shopt -s nullglob - for package in pkgs/*/*.conda; do + [ -f uploads.txt ] || { echo "No status of the recipes: nothing to upload"; exit 1; } + while read -r package; do # the upload task runs in distros//work pixi run rs "$DISTRO" upload "../../../$package" - done + done < uploads.txt + # the job's log has it too; the artifact is for failures, keeping the run's artifacts + # to about one per job - name: Upload build log - if: ${{ always() && steps.build-recipes.outcome != 'skipped' }} + if: ${{ failure() && steps.build-recipes.outcome != 'skipped' }} uses: actions/upload-artifact@v6 with: name: build-log-${{ inputs.distro }}-${{ inputs.platform }}-${{ inputs.job }}-${{ github.run_attempt }} diff --git a/.github/workflows/staged-build.yaml b/.github/workflows/staged-build.yaml index c7f254620..5c7b25304 100644 --- a/.github/workflows/staged-build.yaml +++ b/.github/workflows/staged-build.yaml @@ -1399,11 +1399,18 @@ jobs: cache-write: false # saved from main only (pixi-cache.yaml) - uses: actions/download-artifact@v7 with: - pattern: pkgs-${{ matrix.distro }}-${{ matrix.platform }}-* - merge-multiple: true - path: ${{ env.CACHE_DIR }} + name: schedule + - name: The stage jobs' artifacts + run: pixi run python tools/staged_build.py artifacts --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" + # by name: download-artifact's pattern only sees the run's first 1,000 artifacts + - name: Download the stage jobs' packages + uses: actions/github-script@v8 + with: + script: | + await require('./tools/download_artifacts.cjs')({ names: process.env.ARTIFACTS.split(' '), dir: 'artifacts', core }) - name: See the packages run: | + pixi run python tools/staged_build.py unpack --zips artifacts --into "$CACHE_DIR" rm -rf "$CACHE_DIR/status" shopt -s nullglob packages=("$CACHE_DIR"/*/*.conda) diff --git a/pixi.lock b/pixi.lock index de1b72f0a..c6940c1eb 100644 --- a/pixi.lock +++ b/pixi.lock @@ -90,7 +90,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -170,7 +170,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -243,7 +243,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/tk-8.6.13-ha6c374e_4.conda - conda: https://prefix.dev/conda-forge/osx-64/yaml-0.2.5-had63097_3.conda - conda: https://prefix.dev/conda-forge/osx-64/zstd-1.5.7-hbc1a06c_7.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -316,7 +316,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - conda: https://prefix.dev/conda-forge/osx-arm64/yaml-0.2.5-h74c22ad_3.conda - conda: https://prefix.dev/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -389,7 +389,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/vcomp14-14.51.36247-habf1de7_41.conda - conda: https://prefix.dev/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://prefix.dev/conda-forge/win-64/zstd-1.5.7-h534d264_7.conda - - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -3289,7 +3289,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 size: 387535 timestamp: 1786599623274 -- pypi: git+https://github.com/EzraBrooks/vinca.git?rev=0a04e395f01c70322f234b582422991ab6928caa#0a04e395f01c70322f234b582422991ab6928caa +- pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 name: vinca version: 0.2.0 requires_dist: diff --git a/pixi.toml b/pixi.toml index b1946a7c8..61e3c2854 100644 --- a/pixi.toml +++ b/pixi.toml @@ -36,7 +36,7 @@ git = "*" [pypi-dependencies] # One vinca for all distributions. -vinca = { git = "https://github.com/EzraBrooks/vinca.git", rev = "0a04e395f01c70322f234b582422991ab6928caa" } +vinca = { git = "https://github.com/EzraBrooks/vinca.git", rev = "3fa01ee61c648a062671e569c2050ed90aa07f05" } # For local development, temporarily use (and regenerate pixi.lock): # vinca = { path = "../vinca", editable = true } diff --git a/tools/download_artifacts.cjs b/tools/download_artifacts.cjs new file mode 100644 index 000000000..21c92dde0 --- /dev/null +++ b/tools/download_artifacts.cjs @@ -0,0 +1,104 @@ +// Downloads artifacts of this workflow run by name, as zip files

/.zip, for +// actions/github-script (staged builds, see tools/staged_build.py): +// +// await require('./tools/download_artifacts.cjs')({ names, dir, core }) +// +// download-artifact's `pattern:` filters the run's artifact list, which ends at 1,000 +// artifacts; looking each one up by name has no such limit. These are the artifact +// service's calls that @actions/artifact makes for a download by name (ListArtifacts with a +// name filter, then GetSignedArtifactURL), with the runtime token only JavaScript actions +// get. Returns the names that weren't found. +const fs = require('fs') +const path = require('path') +const { Readable } = require('stream') +const { pipeline } = require('stream/promises') + +const SERVICE = 'github.actions.results.api.v1.ArtifactService' +const PARALLEL = 8 +const ATTEMPTS = 5 +const RETRY_STATUS = [429, 500, 502, 503, 504] + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +// the run's and this job's ids in the artifact service, from the token's Actions.Results scope +function backendIds(token) { + const claims = JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString()) + for (const scope of (claims.scp || '').split(' ')) { + const [name, run, job] = scope.split(':') + if (name === 'Actions.Results' && run && job) { + return { workflow_run_backend_id: run, workflow_job_run_backend_id: job } + } + } + throw new Error('ACTIONS_RUNTIME_TOKEN has no Actions.Results scope') +} + +// request(): a fetch() response, read by use(response); network errors and the +// service's transient statuses are retried, waiting longer each time +async function retrying(what, request, use, core) { + for (let attempt = 1; ; attempt++) { + let error + let wait = 5000 * 2 ** (attempt - 1) + let response + try { + response = await request() + } catch (e) { + error = e.cause ? `${e.message} (${e.cause.message || e.cause})` : e.message + } + if (response) { + if (response.ok) { + try { + return await use(response) + } catch (e) { + error = e.message + } + } else { + error = `${response.status} ${await response.text()}` + if (!RETRY_STATUS.includes(response.status)) throw new Error(`${what}: ${error}`) + wait = Number(response.headers.get('retry-after')) * 1000 || wait + } + } + if (attempt === ATTEMPTS) throw new Error(`${what}: ${error} (after ${ATTEMPTS} attempts)`) + core.info(`${what}: ${error}; retrying in ${wait / 1000} s`) + await sleep(wait) + } +} + +module.exports = async function download({ names, dir, core }) { + const token = process.env.ACTIONS_RUNTIME_TOKEN + const results = process.env.ACTIONS_RESULTS_URL + if (!token || !results) throw new Error('ACTIONS_RUNTIME_TOKEN and ACTIONS_RESULTS_URL are only set in JavaScript actions') + const ids = backendIds(token) + const origin = new URL(results).origin + + const call = (method, body) => retrying(method, () => fetch(`${origin}/twirp/${SERVICE}/${method}`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }), (response) => response.json(), core) + + fs.mkdirSync(dir, { recursive: true }) + const missing = [] + const todo = [...new Set(names)] + const worker = async () => { + for (let name; (name = todo.shift()) !== undefined;) { + const { artifacts = [] } = await call('ListArtifacts', { ...ids, name_filter: name }) + if (!artifacts.length) { + missing.push(name) + continue + } + // the newest, as @actions/artifact takes (re-runs can upload a name again) + const artifact = artifacts.sort((a, b) => Number(b.database_id) - Number(a.database_id))[0] + const { signed_url: url } = await call('GetSignedArtifactURL', { + workflow_run_backend_id: artifact.workflow_run_backend_id, + workflow_job_run_backend_id: artifact.workflow_job_run_backend_id, + name: artifact.name, + }) + await retrying(`download ${name}`, () => fetch(url), (response) => + pipeline(Readable.fromWeb(response.body), fs.createWriteStream(path.join(dir, `${name}.zip`))), core) + core.info(`${name}: ${artifact.size} bytes`) + } + } + await Promise.all(Array.from({ length: PARALLEL }, worker)) + if (missing.length) core.info(`Not found: ${missing.sort().join(' ')}`) + return missing +} diff --git a/tools/robostack.py b/tools/robostack.py index b35d97113..07fed69cb 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -112,6 +112,8 @@ def run_tee(cmd: list[str], cwd: Path) -> tuple[int, bytes]: b"HTTP status client error (409 Conflict)", b"HTTP status server error") NETWORK_RETRIES = 4 NETWORK_WAIT = 60 # doubled on each retry: 1, 2, 4, 8 minutes +# with --continue-on-failure, rattler-build exits with 0 after a failed build or test +FAILED_MESSAGES = (b"Build failed for ", b"Test failed for ") # a build job's recipes side by side, with a shared jobserver keeping them at the runner's @@ -181,11 +183,12 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int: channel's index to catch up with the previous stage's uploads (at most INDEX_LAG_RETRIES times), or when the runner's network failed (at most NETWORK_RETRIES times, waiting longer each time). show(line) prints a line instead of - writing it as is.""" + writing it as is. A build or test that failed under --continue-on-failure fails it + too; a non-zero exit code of the command itself is printed (" exited with N").""" index_retries = network_retries = 0 while True: print("+", " ".join(cmd), f" (in {cwd.relative_to(ROOT)})", flush=True) - missing = network = False + missing = network = failed = False with subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc: for line in proc.stdout: if show: @@ -195,7 +198,11 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int: sys.stdout.buffer.flush() missing = missing or any(m in line.lower() for m in INDEX_LAG_MESSAGES) network = network or any(m in line for m in NETWORK_MESSAGES) - if proc.returncode == 0: + failed = failed or any(m in line for m in FAILED_MESSAGES) + if proc.returncode: + print(f"{Path(cmd[0]).name} exited with {proc.returncode}", file=sys.stderr, flush=True) + returncode = proc.returncode or int(failed) + if returncode == 0: return 0 if network and network_retries < NETWORK_RETRIES: wait = NETWORK_WAIT * 2 ** network_retries @@ -209,7 +216,7 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int: f"(retry {index_retries} of {INDEX_LAG_RETRIES})", file=sys.stderr, flush=True) time.sleep(INDEX_LAG_WAIT) else: - return proc.returncode + return returncode # --------------------------------------------------------------------------- # diff --git a/tools/staged_build.py b/tools/staged_build.py index 75d6b559f..58bcfcc07 100644 --- a/tools/staged_build.py +++ b/tools/staged_build.py @@ -3,6 +3,10 @@ python tools/staged_build.py plan --distro --platform

--os --cache-dir

--out python tools/staged_build.py schedule --out ... python tools/staged_build.py job --schedule --distro --platform

--job + python tools/staged_build.py artifacts --schedule --distro --platform

+ python tools/staged_build.py unpack --zips

--into + python tools/staged_build.py ready --schedule --distro --platform

--job --channel

+ python tools/staged_build.py status --schedule --distro --platform

--job --log --out

python tools/staged_build.py workflow [--check] .github/workflows/staged-build.yaml builds the given distributions and platforms in @@ -18,11 +22,16 @@ matrix limit) and writes the schedule every stage job reads its recipes from. - `job` is a stage job's recipes and the artifacts it needs: the packages of the jobs that build its requirements (vinca's `needs`, and theirs), and the restored cache. + tools/download_artifacts.cjs downloads them by name (`artifacts` lists a distribution + and platform's for collecting them), `unpack` merges them into one directory. +- `ready` leaves out the job's recipes that need a recipe that wasn't built (vinca's + `requires`, in this job or the ones it needs), `status` records how each recipe went. - `workflow` writes staged-build.yaml (`rs check` checks that it is up to date). -Packages pass between jobs as artifacts, pkgs---, each with a -status/ file (success or failure): a job whose requirements failed fails too, but -the other distributions and platforms go on. +Packages pass between jobs as artifacts, pkgs---, each with +status/ (success, or failure when any of its recipes wasn't built) and +status/.json ({recipe: success, failed, skipped or blocked}): a recipe that failed +holds back only the recipes that need it, and the other distributions and platforms go on. """ from __future__ import annotations @@ -30,8 +39,10 @@ import argparse import json import os +import re import shutil import sys +import zipfile from collections import Counter from pathlib import Path @@ -70,6 +81,7 @@ def fold(stages: list[list[dict]], slots: int = SLOTS) -> list[list[dict]]: "name": f"{len(rest)} jobs of stages {slots - 1}+", "recipes": " ".join(job["recipes"] for job in rest), "needs": sorted({n for job in rest for n in job["needs"]} - ids), + "requires": {r: deps for job in rest for r, deps in job.get("requires", {}).items()}, } return [*stages[:slots - 1], [tail]] @@ -108,6 +120,7 @@ def merge_smallest(stages: list[list[dict]], k: int) -> None: a["recipes"] = f"{a['recipes']} {b['recipes']}" a["name"] = f"{a['name']} {b['name']}" a["needs"] = sorted(set(a["needs"]) | set(b["needs"])) + a["requires"] = {**a.get("requires", {}), **b.get("requires", {})} stages[k].remove(b) for stage in stages[k + 1:]: for job in stage: @@ -117,8 +130,8 @@ def merge_smallest(stages: list[list[dict]], k: int) -> None: def schedule(plans: list[dict], slots: int = SLOTS, max_matrix: int = MAX_MATRIX) -> tuple[list[list[dict]], dict]: """The slots' matrix entries, and each distribution and platform's jobs by id - ({"/": {id: {"recipes", "needs"}}}). A slot with more than - max_matrix jobs has the smallest jobs of its largest plans merged.""" + ({"/": {id: {"recipes", "needs", "requires"}}}). A slot with more + than max_matrix jobs has the smallest jobs of its largest plans merged.""" entries = [] for k in range(slots): while sum(len(p["stages"][k]) for p in plans if k < len(p["stages"])) > max_matrix: @@ -131,7 +144,8 @@ def schedule(plans: list[dict], slots: int = SLOTS, max_matrix: int = MAX_MATRIX ]) while entries and not entries[-1]: entries.pop() - jobs = {f"{p['distro']}/{p['platform']}": {job["id"]: {"recipes": job["recipes"], "needs": job["needs"]} + jobs = {f"{p['distro']}/{p['platform']}": {job["id"]: {"recipes": job["recipes"], "needs": job["needs"], + "requires": job.get("requires", {})} for stage in p["stages"] for job in stage} for p in plans} return entries, jobs @@ -152,10 +166,65 @@ def job_needs(jobs: dict[str, dict], job: str) -> list[str]: return sorted(seen) -def pattern(distro: str, platform: str, jobs: list[str]) -> str: - """download-artifact's pattern (minimatch) for these jobs' artifacts.""" - names = ["cache", *jobs] - return artifact(distro, platform, names[0] if len(names) == 1 else "{" + ",".join(names) + "}") +def unpack(zips: Path, into: Path) -> None: + """Extract the downloaded artifacts (tools/download_artifacts.cjs) into one directory.""" + into.mkdir(parents=True, exist_ok=True) + for z in sorted(zips.glob("*.zip")): + with zipfile.ZipFile(z) as f: + f.extractall(into) + z.unlink() + + +def needed_status(jobs: dict[str, dict], needed: list[str], channel: Path) -> dict[str, str]: + """How each recipe of the needed jobs went, from their status/.json; a job + without one (its artifact missing: it didn't get to the end) built none.""" + status = {} + for j in needed: + f = channel / "status" / f"{j}.json" + recipes = json.loads(f.read_text()) if f.is_file() else {} + for r in jobs[j]["recipes"].split(): + status[r] = recipes.get(r, "not built") + return status + + +def blocked(job: dict, status: dict[str, str]) -> dict[str, str]: + """The job's recipes that can't be built, each with the requirement it lacks: one + that the needed jobs didn't build, or one of the job's own recipes held back.""" + out: dict[str, str] = {} + changed = True + while changed: + changed = False + for r in job["recipes"].split(): + if r in out: + continue + lacking = [q for q in job["requires"].get(r, []) if q in out or status.get(q, "success") != "success"] + if lacking: + out[r] = lacking[0] + changed = True + return out + + +def outcomes(recipes: list[str], held: dict[str, str], log: str | None) -> dict[str, str]: + """How each recipe of the job went: blocked (held back), else from the last attempt in + rattler-build's log (rs build-ci with --continue-on-failure=skip-dependents): failed + (its build or test), skipped (it needs one that failed in this job), or success. When + rattler-build itself failed (or there's no log), every recipe that isn't blocked failed.""" + attempt = re.split(r"^\+ rattler-build ", log, flags=re.M)[-1] if log else "" + aborted = not attempt or re.search(r"^rattler-build exited with ", attempt, re.M) + name = lambda identifier: identifier.rsplit("-", 2)[0] # noqa: E731 (name-version-build) + failed = {name(i) for i in re.findall(r"(?:Build|Test) failed for (\S+): ", attempt)} + skipped = {name(i) for i in re.findall(r"Skipping (\S+) because its dependency \S+ failed to build", attempt)} + out = {} + for r in recipes: + if r in held: + out[r] = "blocked" + elif r in failed or aborted: + out[r] = "failed" + elif r in skipped: + out[r] = "skipped" + else: + out[r] = "success" + return out def workflow() -> str: @@ -189,11 +258,22 @@ def main() -> int: s = sub.add_parser("schedule", help="the plans in the stage slots, as GitHub outputs") s.add_argument("--out", required=True, type=Path) s.add_argument("plans", nargs="*", type=Path) - j = sub.add_parser("job", help="a stage job's RECIPES, NEEDED and NEEDS_PATTERN, as GitHub environment") - j.add_argument("--schedule", required=True, type=Path) - j.add_argument("--distro", required=True) - j.add_argument("--platform", required=True) - j.add_argument("--job", required=True) + j = sub.add_parser("job", help="a stage job's RECIPES, NEEDED and NEEDED_ARTIFACTS, as GitHub environment") + a = sub.add_parser("artifacts", help="a distribution and platform's artifacts, ARTIFACTS, as GitHub environment") + r = sub.add_parser("ready", help="the job's RECIPES whose requirements were built, as GitHub environment") + t = sub.add_parser("status", help="how each of the job's recipes went, status/.json in --out") + for p in (j, a, r, t): + p.add_argument("--schedule", required=True, type=Path) + p.add_argument("--distro", required=True) + p.add_argument("--platform", required=True) + for p in (j, r, t): + p.add_argument("--job", required=True) + r.add_argument("--channel", required=True, type=Path, help="the needed jobs' artifacts, unpacked") + t.add_argument("--log", required=True, type=Path, help="the build's output (rs build-ci)") + t.add_argument("--out", required=True, type=Path, help="the job's artifact") + u = sub.add_parser("unpack", help="extract downloaded artifacts into one directory") + u.add_argument("--zips", required=True, type=Path) + u.add_argument("--into", required=True, type=Path) w = sub.add_parser("workflow", help=f"write {WORKFLOW.relative_to(rs.ROOT)}") w.add_argument("--check", action="store_true", help="only check that it is up to date") args = parser.parse_args() @@ -231,13 +311,56 @@ def main() -> int: print(f"{sum(map(len, entries))} jobs in {len(entries)} stages: " + ", ".join(str(len(e)) for e in entries)) return 0 + if args.command == "unpack": + unpack(args.zips, args.into) + return 0 + jobs = json.loads(args.schedule.read_text())[f"{args.distro}/{args.platform}"] + if args.command == "artifacts": + names = [artifact(args.distro, args.platform, j) for j in ["cache", *jobs]] + write_lines("GITHUB_ENV", [f"ARTIFACTS={' '.join(names)}"]) + print(f"{len(names)} artifacts") + return 0 + + job = jobs[args.job] needed = job_needs(jobs, args.job) - lines = [f"RECIPES={jobs[args.job]['recipes']}", f"NEEDED={' '.join(needed)}", - f"NEEDS_PATTERN={pattern(args.distro, args.platform, needed)}"] - write_lines("GITHUB_ENV", lines) - print("\n".join(lines)) - return 0 + held_file = Path(f"blocked-{args.job}.json") + if args.command == "job": + lines = [f"RECIPES={job['recipes']}", f"NEEDED={' '.join(needed)}", + "NEEDED_ARTIFACTS=" + " ".join(artifact(args.distro, args.platform, j) for j in ["cache", *needed])] + write_lines("GITHUB_ENV", lines) + print("\n".join(lines)) + return 0 + + if args.command == "ready": + status = needed_status(jobs, needed, args.channel) + held = blocked(job, status) + held_file.write_text(json.dumps(held) + "\n") + for r, q in held.items(): + print(f"{r}: not built, as it needs {q} ({status.get(q, 'held back in this job')})") + ready = [r for r in job["recipes"].split() if r not in held] + write_lines("GITHUB_ENV", [f"RECIPES={' '.join(ready)}"]) + print(f"{len(ready)} of {len(job['recipes'].split())} recipes to build: {' '.join(ready)}") + return 0 + + held = json.loads(held_file.read_text()) if held_file.is_file() else {} + log = args.log.read_text(encoding="utf-8", errors="replace") if args.log.is_file() else None + result = outcomes(job["recipes"].split(), held, log) + (args.out / "status").mkdir(parents=True, exist_ok=True) + (args.out / "status" / f"{args.job}.json").write_text(json.dumps(result, indent=1) + "\n") + ok = all(s == "success" for s in result.values()) + (args.out / "status" / args.job).write_text(("success" if ok else "failure") + "\n") + lines = [f"| {r} | {s}{f' (needs {held[r]})' if r in held else ''} |" for r, s in result.items() if s != "success"] + for line in lines: + print(line.strip("| ").replace(" | ", ": ")) + if lines: + write_lines("GITHUB_STEP_SUMMARY", ["| recipe | not built |", "|---|---|", *lines, ""]) + # the packages of the recipes that built, to upload (releases) + built = {r for r, s in result.items() if s == "success"} + Path("uploads.txt").write_text("".join(f"{p.as_posix()}\n" for p in sorted(args.out.glob("*/*.conda")) + if p.name.rsplit("-", 2)[0] in built)) + print(f"{len(built)} of {len(result)} recipes built") + return 0 if ok else 1 SETUP = """\ @@ -435,11 +558,18 @@ def main() -> int: """ + SETUP + """\ - uses: actions/download-artifact@v7 with: - pattern: pkgs-${{ matrix.distro }}-${{ matrix.platform }}-* - merge-multiple: true - path: ${{ env.CACHE_DIR }} + name: schedule + - name: The stage jobs' artifacts + run: pixi run python tools/staged_build.py artifacts --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" + # by name: download-artifact's pattern only sees the run's first 1,000 artifacts + - name: Download the stage jobs' packages + uses: actions/github-script@v8 + with: + script: | + await require('./tools/download_artifacts.cjs')({ names: process.env.ARTIFACTS.split(' '), dir: 'artifacts', core }) - name: See the packages run: | + pixi run python tools/staged_build.py unpack --zips artifacts --into "$CACHE_DIR" rm -rf "$CACHE_DIR/status" shopt -s nullglob packages=("$CACHE_DIR"/*/*.conda) diff --git a/tools/tests/test_staged_build.py b/tools/tests/test_staged_build.py index 916060022..fb4bb42d8 100644 --- a/tools/tests/test_staged_build.py +++ b/tools/tests/test_staged_build.py @@ -1,5 +1,7 @@ """tools/staged_build.py: the staged build's plans, schedule and stage slots.""" +import sys + import yaml import robostack as rs @@ -53,16 +55,19 @@ def test_schedule_shares_the_slots_between_distributions_and_platforms(): ("lyrical", "win-64", "stage_0_job_1")], [("jazzy", "linux-64", "stage_1_job_0")], ] - assert jobs["lyrical/win-64"]["stage_0_job_1"] == {"recipes": "ros2-c", "needs": []} + assert jobs["lyrical/win-64"]["stage_0_job_1"] == {"recipes": "ros2-c", "needs": [], "requires": {}} def test_schedule_merges_jobs_past_the_matrix_limit(): stages = [[job("a", "ros2-a"), job("b", "ros2-b ros2-bb"), job("c", "ros2-c")], [job("d", "ros2-d", ["c"])]] + stages[0][2]["requires"] = {"ros2-c": []} + stages[1][0]["requires"] = {"ros2-d": ["ros2-c"]} entries, jobs = sb.schedule([plan("jazzy", "linux-64", stages)], max_matrix=2) assert [e["job"] for e in entries[0]] == ["a", "b"] assert jobs["jazzy/linux-64"]["a"]["recipes"] == "ros2-a ros2-c" + assert jobs["jazzy/linux-64"]["a"]["requires"] == {"ros2-c": []} assert jobs["jazzy/linux-64"]["d"]["needs"] == ["a"] @@ -70,8 +75,80 @@ def test_a_job_needs_its_requirements_jobs_and_theirs(): jobs = {"a": {"needs": []}, "b": {"needs": ["a"]}, "c": {"needs": []}, "d": {"needs": ["b"]}} assert sb.job_needs(jobs, "d") == ["a", "b"] - assert sb.pattern("jazzy", "linux-64", ["a", "b"]) == "pkgs-jazzy-linux-64-{cache,a,b}" - assert sb.pattern("jazzy", "linux-64", []) == "pkgs-jazzy-linux-64-cache" + + +def test_a_failed_recipe_blocks_only_the_recipes_that_need_it(tmp_path): + jobs = { + "s0": {"recipes": "ros2-zstd ros2-mola", "needs": [], "requires": {}}, + "s1": {"recipes": "ros2-a ros2-b ros2-c ros2-d", "needs": ["s0"], + "requires": {"ros2-a": ["ros2-zstd"], "ros2-b": ["ros2-mola"], "ros2-c": ["ros2-b"], "ros2-d": ["ros2-x"]}}, + "s2": {"recipes": "ros2-x", "needs": [], "requires": {}}, + } + (tmp_path / "status").mkdir() + (tmp_path / "status" / "s0.json").write_text('{"ros2-zstd": "success", "ros2-mola": "failed"}') + + # s2's artifact is missing: it didn't finish + status = sb.needed_status(jobs, ["s0", "s2"], tmp_path) + + assert status == {"ros2-zstd": "success", "ros2-mola": "failed", "ros2-x": "not built"} + # ros2-c needs ros2-b, held back in the same job + assert sb.blocked(jobs["s1"], status) == {"ros2-b": "ros2-mola", "ros2-c": "ros2-b", "ros2-d": "ros2-x"} + + +LOG = """\ ++ rattler-build build --recipe ./recipes/ros2-a (in distros/lyrical/work) +10:00:00 [ros2-a] Build failed for ros2-a-1.0.0-np2py312h0_31: network +rattler-build exited with 1 +Network error; retrying in 60 s (retry 1 of 4) ++ rattler-build build --recipe ./recipes/ros2-a (in distros/lyrical/work) +10:01:00 [ros2-a] Archive written to 'output/linux-64/ros2-a-1.0.0-np2py312h0_31.conda' +10:02:00 [ros2-mola] Build failed for ros2-mola-3.3.0-np2py312hdd89193_31: Script failed to execute +10:02:00 [-] Skipping ros-lyrical-mola-3.3.0-h9490d1a_31 because its dependency ros2-mola failed to build +10:03:00 [ros2-t] Test failed for ros2-t-1.0.0-h0_31: import failed +""" + + +def test_each_recipe_s_outcome_from_the_last_attempt(): + recipes = ["ros2-a", "ros2-mola", "ros-lyrical-mola", "ros2-t", "ros2-held"] + + assert sb.outcomes(recipes, {"ros2-held": "ros2-x"}, LOG) == { + "ros2-a": "success", "ros2-mola": "failed", "ros-lyrical-mola": "skipped", "ros2-t": "failed", + "ros2-held": "blocked", + } + + +def test_every_recipe_failed_when_rattler_build_itself_did(): + log = "+ rattler-build build\n10:00:00 [ros2-a] Archive written to 'x'\nrattler-build exited with 1\n" + + assert sb.outcomes(["ros2-a", "ros2-b"], {}, log) == {"ros2-a": "failed", "ros2-b": "failed"} + assert sb.outcomes(["ros2-a", "ros2-b"], {"ros2-b": "ros2-x"}, None) == {"ros2-a": "failed", "ros2-b": "blocked"} + + +def test_unpack_merges_the_artifacts(tmp_path): + import zipfile + + zips = tmp_path / "zips" + zips.mkdir() + for name, files in {"pkgs-a": ["linux-64/ros2-a-1-0.conda", "status/a"], "pkgs-b": ["noarch/ros2-b-1-0.conda", "status/b"]}.items(): + with zipfile.ZipFile(zips / f"{name}.zip", "w") as z: + for f in files: + z.writestr(f, "x") + + sb.unpack(zips, tmp_path / "channel") + + assert sorted(p.relative_to(tmp_path / "channel").as_posix() for p in (tmp_path / "channel").rglob("*") if p.is_file()) == [ + "linux-64/ros2-a-1-0.conda", "noarch/ros2-b-1-0.conda", "status/a", "status/b"] + assert not list(zips.iterdir()) + + +def test_a_build_that_failed_under_continue_on_failure_fails(monkeypatch, tmp_path, capsys): + script = "import sys; print('{\"fields\": {\"message\": \"Build failed for ros2-a-1-0: x\"}}')" + monkeypatch.setattr(rs, "ROOT", tmp_path) + + assert rs.run_build([sys.executable, "-c", script], tmp_path) == 1 + assert rs.run_build([sys.executable, "-c", "print('ok')"], tmp_path) == 0 + assert rs.run_build([sys.executable, "-c", "raise SystemExit(3)"], tmp_path) == 3 + assert "exited with 3" in capsys.readouterr().err def test_every_slot_runs_after_the_one_before(): From 8510b7f35fd38a7a58174949bc76401a26989299 Mon Sep 17 00:00:00 2001 From: Ezra Brooks Date: Fri, 13 Nov 2026 08:46:21 -0700 Subject: [PATCH 6/6] Staged builds: a recipe that only runs with one that failed in its job isn't built either rattler-build's skip-dependents skips the outputs that need a failed one to build, not the ones that only run with it, e.g. ros-- of ros2-. --- tools/staged_build.py | 16 +++++++++++++--- tools/tests/test_staged_build.py | 12 +++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tools/staged_build.py b/tools/staged_build.py index 58bcfcc07..cbdd5c98e 100644 --- a/tools/staged_build.py +++ b/tools/staged_build.py @@ -204,11 +204,14 @@ def blocked(job: dict, status: dict[str, str]) -> dict[str, str]: return out -def outcomes(recipes: list[str], held: dict[str, str], log: str | None) -> dict[str, str]: +def outcomes(job: dict, held: dict[str, str], log: str | None) -> dict[str, str]: """How each recipe of the job went: blocked (held back), else from the last attempt in rattler-build's log (rs build-ci with --continue-on-failure=skip-dependents): failed (its build or test), skipped (it needs one that failed in this job), or success. When - rattler-build itself failed (or there's no log), every recipe that isn't blocked failed.""" + rattler-build itself failed (or there's no log), every recipe that isn't blocked failed. + rattler-build skips only the outputs that need a failed one to build, so one that only + runs with it (e.g. ros--, of ros2-) counts as skipped here.""" + recipes = job["recipes"].split() attempt = re.split(r"^\+ rattler-build ", log, flags=re.M)[-1] if log else "" aborted = not attempt or re.search(r"^rattler-build exited with ", attempt, re.M) name = lambda identifier: identifier.rsplit("-", 2)[0] # noqa: E731 (name-version-build) @@ -224,6 +227,13 @@ def outcomes(recipes: list[str], held: dict[str, str], log: str | None) -> dict[ out[r] = "skipped" else: out[r] = "success" + changed = True + while changed: + changed = False + for r in recipes: + if out[r] == "success" and any(out.get(q, "success") != "success" for q in job["requires"].get(r, [])): + out[r] = "skipped" + changed = True return out @@ -345,7 +355,7 @@ def main() -> int: held = json.loads(held_file.read_text()) if held_file.is_file() else {} log = args.log.read_text(encoding="utf-8", errors="replace") if args.log.is_file() else None - result = outcomes(job["recipes"].split(), held, log) + result = outcomes(job, held, log) (args.out / "status").mkdir(parents=True, exist_ok=True) (args.out / "status" / f"{args.job}.json").write_text(json.dumps(result, indent=1) + "\n") ok = all(s == "success" for s in result.values()) diff --git a/tools/tests/test_staged_build.py b/tools/tests/test_staged_build.py index fb4bb42d8..d6d82fdc5 100644 --- a/tools/tests/test_staged_build.py +++ b/tools/tests/test_staged_build.py @@ -109,19 +109,21 @@ def test_a_failed_recipe_blocks_only_the_recipes_that_need_it(tmp_path): def test_each_recipe_s_outcome_from_the_last_attempt(): - recipes = ["ros2-a", "ros2-mola", "ros-lyrical-mola", "ros2-t", "ros2-held"] + job = {"recipes": "ros2-a ros2-mola ros-lyrical-mola ros2-t ros-lyrical-t ros2-held", "requires": {"ros-lyrical-t": ["ros2-t"]}} - assert sb.outcomes(recipes, {"ros2-held": "ros2-x"}, LOG) == { + # ros-lyrical-t only runs with ros2-t: rattler-build built it + assert sb.outcomes(job, {"ros2-held": "ros2-x"}, LOG) == { "ros2-a": "success", "ros2-mola": "failed", "ros-lyrical-mola": "skipped", "ros2-t": "failed", - "ros2-held": "blocked", + "ros-lyrical-t": "skipped", "ros2-held": "blocked", } def test_every_recipe_failed_when_rattler_build_itself_did(): log = "+ rattler-build build\n10:00:00 [ros2-a] Archive written to 'x'\nrattler-build exited with 1\n" + job = {"recipes": "ros2-a ros2-b", "requires": {}} - assert sb.outcomes(["ros2-a", "ros2-b"], {}, log) == {"ros2-a": "failed", "ros2-b": "failed"} - assert sb.outcomes(["ros2-a", "ros2-b"], {"ros2-b": "ros2-x"}, None) == {"ros2-a": "failed", "ros2-b": "blocked"} + assert sb.outcomes(job, {}, log) == {"ros2-a": "failed", "ros2-b": "failed"} + assert sb.outcomes(job, {"ros2-b": "ros2-x"}, None) == {"ros2-a": "failed", "ros2-b": "blocked"} def test_unpack_merges_the_artifacts(tmp_path):