diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..6c8e9990 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,65 @@ +# Release builds in vinca-gha's stages (staged-build.yaml), uploading to each +# distribution's channel (or the repository variable ROBOSTACK_UPLOAD_CHANNEL): the +# packages missing there are built and uploaded. Started by hand while the build +# branches (main.yaml) still build the releases. +# +# prefix.dev uploads use trusted publishing: the channels' Repository Access must allow +# .github/workflows/stage-job.yaml (called from this workflow). +name: Release build + +on: + workflow_dispatch: + inputs: + distros: + description: Space-separated distributions (default all) + required: false + platforms: + description: Space-separated platforms, or linux / osx / win (default all) + required: false + +concurrency: + group: release-build + cancel-in-progress: false + +permissions: + contents: read + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + distros: ${{ steps.plan.outputs.distros }} + targets: ${{ steps.plan.outputs.targets }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - id: plan + env: + DISTROS: ${{ inputs.distros }} + PLATFORMS: ${{ inputs.platforms }} + run: | + # the same selection as a pull request's build: / build: labels + labels=$(pixi run python -c 'import json,os; print(json.dumps([f"build:{n}" for n in (os.environ["DISTROS"] + " " + os.environ["PLATFORMS"]).split()]))') + echo "distros=$(pixi run python tools/robostack.py changed-distros --base HEAD --all --labels "$labels")" | tee -a "$GITHUB_OUTPUT" + echo "targets=$(pixi run python tools/robostack.py build-targets --labels "$labels")" | tee -a "$GITHUB_OUTPUT" + + build: + needs: plan + if: needs.plan.outputs.distros != '[]' + permissions: + contents: read + id-token: write # prefix.dev trusted publishing + uses: ./.github/workflows/staged-build.yaml + secrets: inherit # ANACONDA_API_TOKEN_ + with: + distros: ${{ needs.plan.outputs.distros }} + targets: ${{ needs.plan.outputs.targets }} + upload: true diff --git a/.github/workflows/stage-job.yaml b/.github/workflows/stage-job.yaml new file mode 100644 index 00000000..406d9aaa --- /dev/null +++ b/.github/workflows/stage-job.yaml @@ -0,0 +1,228 @@ +# One build job of a staged build (staged-build.yaml, tools/staged_build.py): builds its +# recipes whose requirements were built, with the packages of the jobs it needs (and the +# restored cache) as a local channel, and passes what it built on as an artifact, +# pkgs---, with status/ and status/.json (each recipe's); +# releases also upload what built to the distribution's channels. +name: Stage job + +on: + workflow_call: + inputs: + distro: + required: true + type: string + platform: + required: true + type: string + os: + required: true + type: string + job: + description: The job's id in the schedule (vinca-gha's stage__job_) + required: true + type: string + upload: + required: true + type: boolean + token_secret: + description: The repository secret with the distribution's anaconda.org token + required: true + type: string + +env: + PYTHONIOENCODING: utf-8 + DISTRO: ${{ inputs.distro }} + PLATFORM: ${{ inputs.platform }} + JOB: ${{ inputs.job }} + ARTIFACT: pkgs-${{ inputs.distro }}-${{ inputs.platform }}-${{ inputs.job }} + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} + # rattler_build() in tools/robostack.py adds it as a channel (relative to the repository) + ROBOSTACK_LOCAL_CHANNEL: pr-channel + # rattler-build writes to distros//work/output, or to CONDA_BLD_PATH (win-64) + OUTPUT_DIR: ${{ inputs.platform == 'win-64' && 'D:/bld' || format('distros/{0}/work/output', inputs.distro) }} + +jobs: + build: + runs-on: ${{ inputs.os }} + timeout-minutes: 350 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + # Re-running failed jobs re-runs every later stage too: the jobs that succeeded + # before keep their packages. + - name: Download this job's packages of an earlier attempt + uses: actions/github-script@v8 + with: + script: | + await require('./tools/download_artifacts.cjs')({ names: [process.env.ARTIFACT], dir: 'earlier', core }) + - name: Built in an earlier attempt? + id: earlier + shell: bash + run: | + # (python3 on Windows can be the Microsoft Store's placeholder) + status=$("$(command -v python || command -v python3)" -c 'import sys, zipfile; print(zipfile.ZipFile(sys.argv[1]).read(sys.argv[2]).decode().strip())' \ + "earlier/$ARTIFACT.zip" "status/$JOB" 2>/dev/null || true) + if [ "$status" == success ]; then + echo "Built in an earlier attempt of this run" + echo "built=true" >> "$GITHUB_OUTPUT" + fi + + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + if: steps.earlier.outputs.built != 'true' + shell: bash + run: git config --global maintenance.auto false && git config --global gc.auto 0 && git config --global core.longpaths true + # pixi's package cache on D:, with the builds' (see the Windows step below) + - name: "pixi's cache on D:" + if: steps.earlier.outputs.built != 'true' && runner.os == 'Windows' + shell: pwsh + run: '"PIXI_CACHE_DIR=D:\pixi-cache" | Out-File -FilePath $env:GITHUB_ENV -Append' + - uses: prefix-dev/setup-pixi@v0.11.0 + if: steps.earlier.outputs.built != 'true' + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + + - name: Windows workarounds (short build path, long paths, PATH and SDK clean-up) + if: steps.earlier.outputs.built != 'true' && runner.os == 'Windows' + shell: pwsh + run: | + # Defender's real-time scan slows down installing and deleting the environments' + # files several times over; the runner is thrown away after the job + Set-MpPreference -DisableRealtimeMonitoring $true + # the builds, rattler's package cache (on the same drive, for hardlinks) and the + # temporary files on D:, about three times as fast as C: on GitHub's runners + "CONDA_BLD_PATH=D:\bld\", "RATTLER_CACHE_DIR=D:\rattler", "TMP=D:\tmp", "TEMP=D:\tmp" | + Out-File -FilePath $env:GITHUB_ENV -Append + New-Item -ItemType Directory -Force -Path D:\bld, D:\rattler, D:\tmp | Out-Null + New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force + Remove-Item -Recurse -Force C:\Strawberry -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0" -ErrorAction SilentlyContinue + + # Spotlight indexes every file the environments and builds write, using up to a + # core or two of the runner's three or four; the runner is thrown away after the job + - name: Turn off Spotlight + if: steps.earlier.outputs.built != 'true' && runner.os == 'macOS' + run: sudo mdutil -a -i off || true + + - name: Clean up PATH + if: steps.earlier.outputs.built != 'true' && runner.os == 'Windows' + uses: egor-tensin/cleanup-path@v5 + with: + dirs: 'C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin' + + - uses: actions/download-artifact@v7 + if: steps.earlier.outputs.built != 'true' + with: + name: schedule + - name: The job's recipes and requirements + if: steps.earlier.outputs.built != 'true' + shell: bash -l {0} + run: pixi run python tools/staged_build.py job --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" --job "$JOB" + + # the plan job's recipes (generated once for all the jobs) + - uses: actions/download-artifact@v7 + if: steps.earlier.outputs.built != 'true' + with: + name: recipes-${{ inputs.distro }}-${{ inputs.platform }} + - name: Prepare the work directory + if: steps.earlier.outputs.built != 'true' + shell: bash -l {0} + run: pixi run rs "$DISTRO" prepare && tar -xf recipes.tar -C "distros/$DISTRO/work" + # by name: download-artifact's pattern only sees the run's first 1,000 artifacts + - name: Download the packages of the jobs this one needs, and the restored cache + if: steps.earlier.outputs.built != 'true' + uses: actions/github-script@v8 + with: + script: | + await require('./tools/download_artifacts.cjs')({ names: process.env.NEEDED_ARTIFACTS.split(' '), dir: 'needed', core }) + # the recipes that need one that wasn't built are left out (and recorded as blocked) + - name: The recipes whose requirements were built + if: steps.earlier.outputs.built != 'true' + shell: bash -l {0} + run: | + pixi run python tools/staged_build.py unpack --zips needed --into pr-channel + pixi run python tools/staged_build.py ready --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" --job "$JOB" --channel pr-channel + rm -rf pr-channel/status + mkdir -p "pr-channel/$PLATFORM" pr-channel/noarch + shopt -s nullglob + packages=(pr-channel/*/*.conda) + echo "${#packages[@]} packages from the jobs this one needs and the cache" + pixi run rattler-index fs pr-channel --force + + - name: Build recipes + id: build-recipes + if: steps.earlier.outputs.built != 'true' && env.RECIPES != '' + shell: bash -l {0} + run: | + set +e + args=() + for recipe in $RECIPES; do args+=(--recipe "./recipes/$recipe"); done + # shorter work dirs for MAX_PATH on Windows + [ "$PLATFORM" == "win-64" ] && args+=(--no-build-id) + # the cache's packages are in pr-channel; build-ci builds independent recipes side by + # side, and after a failed one goes on with those that don't need it + pixi run rs "$DISTRO" build-ci "${args[@]}" --skip-existing=all --continue-on-failure=skip-dependents 2>&1 | tee rattler-build.log + exit "${PIPESTATUS[0]}" + + - name: Summarize build result + if: ${{ always() && steps.build-recipes.outcome != 'skipped' }} + shell: bash -l {0} + run: | + pixi run python tools/summarize_rattler_build.py --log rattler-build.log \ + --platform "$DISTRO $PLATFORM" --outcome "${{ steps.build-recipes.outcome }}" >> "$GITHUB_STEP_SUMMARY" + + # also after a failure: what was built goes into the pull request's cache, and the + # status tells the later jobs which recipes they can build + - name: Collect the packages + if: ${{ (success() || failure()) && steps.earlier.outputs.built != 'true' }} + shell: bash -l {0} + run: | + shopt -s nullglob + mkdir -p pkgs/status + for subdir in "$PLATFORM" noarch; do + files=("$OUTPUT_DIR/$subdir"/*.conda) + if (( ${#files[@]} )); then mkdir -p "pkgs/$subdir" && cp "${files[@]}" "pkgs/$subdir/"; fi + done + (cd pkgs && printf '%s\n' ./*/*.conda) + pixi run python tools/staged_build.py status --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" --job "$JOB" \ + --log rattler-build.log --out pkgs + - name: Upload the packages + if: ${{ (success() || failure()) && steps.earlier.outputs.built != 'true' }} + uses: actions/upload-artifact@v6 + with: + name: ${{ env.ARTIFACT }} + path: pkgs + compression-level: 0 # .conda files are compressed + overwrite: true # re-runs of a failed job + retention-days: 3 + + # The packages of the recipes that built (uploads.txt, from the status step). + # prefix.dev: trusted publishing (the caller grants id-token: write); anaconda.org: + # the distribution's token. Both skip packages that are already there. + - name: Upload to the distribution's channels + if: ${{ inputs.upload && (success() || failure()) && steps.build-recipes.outcome != 'skipped' }} + shell: bash -l {0} + env: + ANACONDA_API_TOKEN: ${{ secrets[inputs.token_secret] }} + run: | + [ -f uploads.txt ] || { echo "No status of the recipes: nothing to upload"; exit 1; } + while read -r package; do + # the upload task runs in distros//work + pixi run rs "$DISTRO" upload "../../../$package" + done < uploads.txt + + # the job's log has it too; the artifact is for failures, keeping the run's artifacts + # to about one per job + - name: Upload build log + if: ${{ failure() && steps.build-recipes.outcome != 'skipped' }} + uses: actions/upload-artifact@v6 + with: + name: build-log-${{ inputs.distro }}-${{ inputs.platform }}-${{ inputs.job }}-${{ github.run_attempt }} + path: rattler-build.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/staged-build.yaml b/.github/workflows/staged-build.yaml new file mode 100644 index 00000000..5c7b2530 --- /dev/null +++ b/.github/workflows/staged-build.yaml @@ -0,0 +1,1479 @@ +# Generated by tools/staged_build.py (python tools/staged_build.py workflow); `pixi run rs check` +# checks that it is up to date. +# +# The given distributions and platforms, built in vinca-gha's stages, one job per batch +# of recipes (stage-job.yaml): pull requests (testpr.yaml) keep the packages as the +# pull request's build cache, releases (release.yaml) upload them. See +# tools/staged_build.py for how the stages become jobs. +name: Staged build + +on: + workflow_call: + inputs: + distros: + description: JSON list of distributions + required: true + type: string + targets: + description: 'JSON list of platforms and their runners, [{"os": ..., "platform": ...}]' + required: true + type: string + upload: + description: Upload the packages to the distributions' channels (else keep them as the pull request's cache) + required: true + type: boolean + +env: + PYTHONIOENCODING: utf-8 + # while uploads go to a test channel, packages already there aren't built again + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} + # the pull request's packages, /*.conda; restored and saved on Linux for every platform + CACHE_DIR: pr-cache + +jobs: + plan: + strategy: + fail-fast: false + matrix: + distro: ${{ fromJSON(inputs.distros) }} + target: ${{ fromJSON(inputs.targets) }} + name: plan ${{ matrix.distro }} ${{ matrix.target.platform }} + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.target.platform }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - name: Generate recipes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi + echo "Recipe generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done + # a tarball: artifacts drop the build scripts' executable bit + - name: Pack the recipes + run: mkdir -p "distros/$DISTRO/work/recipes" && tar -cf recipes.tar -C "distros/$DISTRO/work" recipes + - name: Upload the recipes + uses: actions/upload-artifact@v6 + with: + name: recipes-${{ matrix.distro }}-${{ matrix.target.platform }} + path: recipes.tar + overwrite: true + retention-days: 7 + - name: Restore build cache + if: ${{ !inputs.upload }} + uses: actions/cache/restore@v5 + with: + path: ${{ env.CACHE_DIR }} + key: staged-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }} + restore-keys: | + staged-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}- + # Temporary rebuild controls live in distros//ci.yaml. + - name: Apply ci.yaml cache controls + run: | + for subdir in "$PLATFORM" noarch; do + mkdir -p "$CACHE_DIR/$subdir" + pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR/$subdir" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" + done + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages restored from the cache" + for package in "${packages[@]}"; do basename "$package"; done + mkdir -p "$CACHE_DIR/status" && echo success > "$CACHE_DIR/status/cache" + - name: Upload the cached packages + uses: actions/upload-artifact@v6 + with: + name: pkgs-${{ matrix.distro }}-${{ matrix.target.platform }}-cache + path: ${{ env.CACHE_DIR }} + compression-level: 0 # .conda files are compressed + overwrite: true + retention-days: 3 + - name: Estimate the build time + run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR/$PLATFORM" --platform "$PLATFORM" + - name: Plan the stages + env: + OS: ${{ matrix.target.os }} + run: pixi run python tools/staged_build.py plan --distro "$DISTRO" --platform "$PLATFORM" --os "$OS" --cache-dir "$CACHE_DIR" --out "plan-$DISTRO-$PLATFORM.json" + - uses: actions/upload-artifact@v6 + with: + name: plan-${{ matrix.distro }}-${{ matrix.target.platform }} + path: plan-${{ matrix.distro }}-${{ matrix.target.platform }}.json + overwrite: true + retention-days: 7 + + # Every plan in the stage slots, and the schedule the stage jobs read their recipes from; + # a failed plan leaves out only its distribution and platform. + schedule: + needs: plan + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + outputs: + count: ${{ steps.schedule.outputs.count }} + combos: ${{ steps.schedule.outputs.combos }} + slot_0: ${{ steps.schedule.outputs.slot_0 }} + slot_1: ${{ steps.schedule.outputs.slot_1 }} + slot_2: ${{ steps.schedule.outputs.slot_2 }} + slot_3: ${{ steps.schedule.outputs.slot_3 }} + slot_4: ${{ steps.schedule.outputs.slot_4 }} + slot_5: ${{ steps.schedule.outputs.slot_5 }} + slot_6: ${{ steps.schedule.outputs.slot_6 }} + slot_7: ${{ steps.schedule.outputs.slot_7 }} + slot_8: ${{ steps.schedule.outputs.slot_8 }} + slot_9: ${{ steps.schedule.outputs.slot_9 }} + slot_10: ${{ steps.schedule.outputs.slot_10 }} + slot_11: ${{ steps.schedule.outputs.slot_11 }} + slot_12: ${{ steps.schedule.outputs.slot_12 }} + slot_13: ${{ steps.schedule.outputs.slot_13 }} + slot_14: ${{ steps.schedule.outputs.slot_14 }} + slot_15: ${{ steps.schedule.outputs.slot_15 }} + slot_16: ${{ steps.schedule.outputs.slot_16 }} + slot_17: ${{ steps.schedule.outputs.slot_17 }} + slot_18: ${{ steps.schedule.outputs.slot_18 }} + slot_19: ${{ steps.schedule.outputs.slot_19 }} + slot_20: ${{ steps.schedule.outputs.slot_20 }} + slot_21: ${{ steps.schedule.outputs.slot_21 }} + slot_22: ${{ steps.schedule.outputs.slot_22 }} + slot_23: ${{ steps.schedule.outputs.slot_23 }} + slot_24: ${{ steps.schedule.outputs.slot_24 }} + slot_25: ${{ steps.schedule.outputs.slot_25 }} + slot_26: ${{ steps.schedule.outputs.slot_26 }} + slot_27: ${{ steps.schedule.outputs.slot_27 }} + slot_28: ${{ steps.schedule.outputs.slot_28 }} + slot_29: ${{ steps.schedule.outputs.slot_29 }} + slot_30: ${{ steps.schedule.outputs.slot_30 }} + slot_31: ${{ steps.schedule.outputs.slot_31 }} + slot_32: ${{ steps.schedule.outputs.slot_32 }} + slot_33: ${{ steps.schedule.outputs.slot_33 }} + slot_34: ${{ steps.schedule.outputs.slot_34 }} + slot_35: ${{ steps.schedule.outputs.slot_35 }} + slot_36: ${{ steps.schedule.outputs.slot_36 }} + slot_37: ${{ steps.schedule.outputs.slot_37 }} + slot_38: ${{ steps.schedule.outputs.slot_38 }} + slot_39: ${{ steps.schedule.outputs.slot_39 }} + slot_40: ${{ steps.schedule.outputs.slot_40 }} + slot_41: ${{ steps.schedule.outputs.slot_41 }} + slot_42: ${{ steps.schedule.outputs.slot_42 }} + slot_43: ${{ steps.schedule.outputs.slot_43 }} + slot_44: ${{ steps.schedule.outputs.slot_44 }} + slot_45: ${{ steps.schedule.outputs.slot_45 }} + slot_46: ${{ steps.schedule.outputs.slot_46 }} + slot_47: ${{ steps.schedule.outputs.slot_47 }} + slot_48: ${{ steps.schedule.outputs.slot_48 }} + slot_49: ${{ steps.schedule.outputs.slot_49 }} + slot_50: ${{ steps.schedule.outputs.slot_50 }} + slot_51: ${{ steps.schedule.outputs.slot_51 }} + slot_52: ${{ steps.schedule.outputs.slot_52 }} + slot_53: ${{ steps.schedule.outputs.slot_53 }} + slot_54: ${{ steps.schedule.outputs.slot_54 }} + slot_55: ${{ steps.schedule.outputs.slot_55 }} + slot_56: ${{ steps.schedule.outputs.slot_56 }} + slot_57: ${{ steps.schedule.outputs.slot_57 }} + slot_58: ${{ steps.schedule.outputs.slot_58 }} + slot_59: ${{ steps.schedule.outputs.slot_59 }} + slot_60: ${{ steps.schedule.outputs.slot_60 }} + slot_61: ${{ steps.schedule.outputs.slot_61 }} + slot_62: ${{ steps.schedule.outputs.slot_62 }} + slot_63: ${{ steps.schedule.outputs.slot_63 }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - uses: actions/download-artifact@v7 + with: + pattern: plan-* + merge-multiple: true + path: plans + - name: Schedule the stages + id: schedule + run: pixi run python tools/staged_build.py schedule --out schedule.json plans/*.json + - uses: actions/upload-artifact@v6 + with: + name: schedule + path: schedule.json + overwrite: true + retention-days: 7 + + stage_0: + needs: schedule + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 0 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_0) }} + name: 0 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_1: + needs: [schedule, stage_0] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 1 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_1) }} + name: 1 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_2: + needs: [schedule, stage_1] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 2 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_2) }} + name: 2 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_3: + needs: [schedule, stage_2] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 3 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_3) }} + name: 3 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_4: + needs: [schedule, stage_3] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 4 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_4) }} + name: 4 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_5: + needs: [schedule, stage_4] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 5 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_5) }} + name: 5 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_6: + needs: [schedule, stage_5] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 6 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_6) }} + name: 6 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_7: + needs: [schedule, stage_6] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 7 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_7) }} + name: 7 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_8: + needs: [schedule, stage_7] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 8 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_8) }} + name: 8 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_9: + needs: [schedule, stage_8] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 9 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_9) }} + name: 9 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_10: + needs: [schedule, stage_9] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 10 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_10) }} + name: 10 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_11: + needs: [schedule, stage_10] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 11 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_11) }} + name: 11 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_12: + needs: [schedule, stage_11] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 12 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_12) }} + name: 12 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_13: + needs: [schedule, stage_12] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 13 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_13) }} + name: 13 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_14: + needs: [schedule, stage_13] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 14 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_14) }} + name: 14 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_15: + needs: [schedule, stage_14] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 15 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_15) }} + name: 15 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_16: + needs: [schedule, stage_15] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 16 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_16) }} + name: 16 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_17: + needs: [schedule, stage_16] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 17 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_17) }} + name: 17 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_18: + needs: [schedule, stage_17] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 18 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_18) }} + name: 18 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_19: + needs: [schedule, stage_18] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 19 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_19) }} + name: 19 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_20: + needs: [schedule, stage_19] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 20 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_20) }} + name: 20 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_21: + needs: [schedule, stage_20] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 21 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_21) }} + name: 21 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_22: + needs: [schedule, stage_21] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 22 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_22) }} + name: 22 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_23: + needs: [schedule, stage_22] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 23 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_23) }} + name: 23 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_24: + needs: [schedule, stage_23] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 24 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_24) }} + name: 24 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_25: + needs: [schedule, stage_24] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 25 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_25) }} + name: 25 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_26: + needs: [schedule, stage_25] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 26 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_26) }} + name: 26 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_27: + needs: [schedule, stage_26] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 27 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_27) }} + name: 27 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_28: + needs: [schedule, stage_27] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 28 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_28) }} + name: 28 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_29: + needs: [schedule, stage_28] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 29 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_29) }} + name: 29 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_30: + needs: [schedule, stage_29] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 30 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_30) }} + name: 30 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_31: + needs: [schedule, stage_30] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 31 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_31) }} + name: 31 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_32: + needs: [schedule, stage_31] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 32 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_32) }} + name: 32 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_33: + needs: [schedule, stage_32] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 33 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_33) }} + name: 33 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_34: + needs: [schedule, stage_33] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 34 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_34) }} + name: 34 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_35: + needs: [schedule, stage_34] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 35 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_35) }} + name: 35 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_36: + needs: [schedule, stage_35] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 36 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_36) }} + name: 36 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_37: + needs: [schedule, stage_36] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 37 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_37) }} + name: 37 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_38: + needs: [schedule, stage_37] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 38 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_38) }} + name: 38 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_39: + needs: [schedule, stage_38] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 39 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_39) }} + name: 39 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_40: + needs: [schedule, stage_39] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 40 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_40) }} + name: 40 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_41: + needs: [schedule, stage_40] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 41 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_41) }} + name: 41 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_42: + needs: [schedule, stage_41] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 42 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_42) }} + name: 42 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_43: + needs: [schedule, stage_42] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 43 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_43) }} + name: 43 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_44: + needs: [schedule, stage_43] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 44 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_44) }} + name: 44 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_45: + needs: [schedule, stage_44] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 45 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_45) }} + name: 45 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_46: + needs: [schedule, stage_45] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 46 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_46) }} + name: 46 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_47: + needs: [schedule, stage_46] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 47 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_47) }} + name: 47 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_48: + needs: [schedule, stage_47] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 48 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_48) }} + name: 48 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_49: + needs: [schedule, stage_48] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 49 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_49) }} + name: 49 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_50: + needs: [schedule, stage_49] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 50 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_50) }} + name: 50 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_51: + needs: [schedule, stage_50] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 51 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_51) }} + name: 51 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_52: + needs: [schedule, stage_51] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 52 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_52) }} + name: 52 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_53: + needs: [schedule, stage_52] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 53 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_53) }} + name: 53 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_54: + needs: [schedule, stage_53] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 54 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_54) }} + name: 54 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_55: + needs: [schedule, stage_54] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 55 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_55) }} + name: 55 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_56: + needs: [schedule, stage_55] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 56 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_56) }} + name: 56 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_57: + needs: [schedule, stage_56] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 57 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_57) }} + name: 57 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_58: + needs: [schedule, stage_57] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 58 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_58) }} + name: 58 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_59: + needs: [schedule, stage_58] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 59 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_59) }} + name: 59 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_60: + needs: [schedule, stage_59] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 60 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_60) }} + name: 60 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_61: + needs: [schedule, stage_60] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 61 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_61) }} + name: 61 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_62: + needs: [schedule, stage_61] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 62 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_62) }} + name: 62 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + stage_63: + needs: [schedule, stage_62] + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > 63 }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_63) }} + name: 63 ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('{"humble":"ANACONDA_API_TOKEN_HUMBLE","jazzy":"ANACONDA_API_TOKEN_JAZZY","lyrical":"ANACONDA_API_TOKEN_LYRICAL","rolling":"ANACONDA_API_TOKEN_ROLLING"}')[matrix.distro] }} + + # Pull requests: the packages of every stage as the pull request's build cache (a new + # push builds only what changed), then the ABI check of what the pull request built. + collect: + needs: [schedule, stage_0, stage_1, stage_2, stage_3, stage_4, stage_5, stage_6, stage_7, stage_8, stage_9, stage_10, stage_11, stage_12, stage_13, stage_14, stage_15, stage_16, stage_17, stage_18, stage_19, stage_20, stage_21, stage_22, stage_23, stage_24, stage_25, stage_26, stage_27, stage_28, stage_29, stage_30, stage_31, stage_32, stage_33, stage_34, stage_35, stage_36, stage_37, stage_38, stage_39, stage_40, stage_41, stage_42, stage_43, stage_44, stage_45, stage_46, stage_47, stage_48, stage_49, stage_50, stage_51, stage_52, stage_53, stage_54, stage_55, stage_56, stage_57, stage_58, stage_59, stage_60, stage_61, stage_62, stage_63] + if: ${{ always() && !inputs.upload && needs.schedule.result == 'success' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.combos) }} + name: collect ${{ matrix.distro }} ${{ matrix.platform }} + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) + - uses: actions/download-artifact@v7 + with: + name: schedule + - name: The stage jobs' artifacts + run: pixi run python tools/staged_build.py artifacts --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" + # by name: download-artifact's pattern only sees the run's first 1,000 artifacts + - name: Download the stage jobs' packages + uses: actions/github-script@v8 + with: + script: | + await require('./tools/download_artifacts.cjs')({ names: process.env.ARTIFACTS.split(' '), dir: 'artifacts', core }) + - name: See the packages + run: | + pixi run python tools/staged_build.py unpack --zips artifacts --into "$CACHE_DIR" + rm -rf "$CACHE_DIR/status" + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages" + for package in "${packages[@]}"; do basename "$package"; done + - name: Save build cache + id: save-cache + uses: actions/cache/save@v5 + with: + path: ${{ env.CACHE_DIR }} + key: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + + # Only the newest cache of a pull request/distribution/platform is restored; the ones + # of earlier runs and attempts only push others out of the repository's 10 GB, so + # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. + - name: Drop this pull request's superseded build caches + if: ${{ steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PREFIX: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}- + KEEP: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + run: | + # cache/save reports success even on upload failure: verify the replacement exists. + if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \ + --jq "if any(.[]; .key == \"$KEEP\") then .[] | select(.key != \"$KEEP\") | .id else error(\"New cache not found\") end"); then + echo "::warning::Could not verify the new build cache; keeping all previous caches." + exit 0 + fi + while read -r id; do + [ -n "$id" ] || continue + pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" + done <<< "$ids" + + # Compare what the pull request built with the released builds, so single packages + # can be bumped without a full rebuild; it reports and never fails the build. + - name: ABI check against the released builds + id: abi-check + if: matrix.platform == 'linux-64' + continue-on-error: true + run: | + pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR/$PLATFORM" --summary "abi-check-$DISTRO.md" + cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" + - name: Upload the ABI check + if: ${{ steps.abi-check.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: abi-check-${{ matrix.distro }} + path: abi-check-${{ matrix.distro }}.md + retention-days: 7 + + # the build branches' workflows, while they still build the releases + - uses: actions/download-artifact@v7 + with: + name: recipes-${{ matrix.distro }}-${{ matrix.platform }} + - name: Prepare the work directory + run: pixi run rs "$DISTRO" prepare && tar -xf recipes.tar -C "distros/$DISTRO/work" + - name: Generate the staged build workflows (catches post-merge problems) + run: | + if [ -z "$(ls "distros/$DISTRO/work/recipes" 2>/dev/null)" ]; then echo "Nothing to build"; exit 0; fi + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi + echo "Workflow generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index c93483fd..3cab799a 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -65,6 +65,9 @@ jobs: echo "distros=$distros" | tee -a "$GITHUB_OUTPUT" # build:linux / build:osx / build:win labels limit the platforms (all without) echo "targets=$(pixi run python tools/robostack.py build-targets --labels "$labels")" | tee -a "$GITHUB_OUTPUT" + # DO NOT MERGE: only lyrical on linux-64 and win-64, to try the staged build + echo 'distros=["lyrical"]' | tee -a "$GITHUB_OUTPUT" + echo 'targets=[{"os": "ubuntu-latest", "platform": "linux-64"}, {"os": "windows-2022", "platform": "win-64"}]' | tee -a "$GITHUB_OUTPUT" check: runs-on: ubuntu-latest @@ -136,226 +139,19 @@ jobs: pixi run rs "$DISTRO" check-orphaned-patches pixi run rs "$DISTRO" check-patches + # The changed distributions on every platform, in vinca-gha's stages, one job per + # batch of recipes (staged-build.yaml, generated by tools/staged_build.py). build: permissions: actions: write # drop this pull request's superseded build caches contents: read needs: plan if: needs.plan.outputs.distros != '[]' - strategy: - fail-fast: false - matrix: - distro: ${{ fromJSON(needs.plan.outputs.distros) }} - target: ${{ fromJSON(needs.plan.outputs.targets) }} # tools/robostack.py BUILD_TARGETS - name: ${{ matrix.distro }} ${{ matrix.target.platform }} - runs-on: ${{ matrix.target.os }} - env: - DISTRO: ${{ matrix.distro }} - PLATFORM: ${{ matrix.target.platform }} - # while uploads go to a test channel, packages already there aren't built again - ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} - # rattler-build writes to distros//work/output, or to CONDA_BLD_PATH (win-64) - CACHE_DIR: ${{ matrix.target.platform == 'win-64' && 'D:/bld/win-64' || format('distros/{0}/work/output/{1}', matrix.distro, matrix.target.platform) }} - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - fetch-depth: 0 - - # git auto-maintenance can leave a lock file in the git clones pixi caches - # (e.g. vinca's), which breaks every later run restoring that cache - - name: Disable git auto-maintenance - run: git config --global maintenance.auto false && git config --global gc.auto 0 && git config --global core.longpaths true - # pixi's package cache on D:, with the builds' (see the Windows step below) - - name: "pixi's cache on D:" - if: runner.os == 'Windows' - shell: pwsh - run: '"PIXI_CACHE_DIR=D:\pixi-cache" | Out-File -FilePath $env:GITHUB_ENV -Append' - - uses: prefix-dev/setup-pixi@v0.11.0 - with: - locked: true # the lock file must match pixi.toml - pixi-version: v0.81.0 - cache-write: false # saved from main only (pixi-cache.yaml) - - - name: Windows workarounds (short build path, long paths, PATH and SDK clean-up) - if: runner.os == 'Windows' - shell: pwsh - run: | - # Defender's real-time scan slows down installing and deleting the environments' - # files several times over; the runner is thrown away after the job - Set-MpPreference -DisableRealtimeMonitoring $true - # the builds, rattler's package cache (on the same drive, for hardlinks) and the - # temporary files on D:, about three times as fast as C: on GitHub's runners - "CONDA_BLD_PATH=D:\bld\", "RATTLER_CACHE_DIR=D:\rattler", "TMP=D:\tmp", "TEMP=D:\tmp" | - Out-File -FilePath $env:GITHUB_ENV -Append - New-Item -ItemType Directory -Force -Path D:\bld, D:\rattler, D:\tmp | Out-Null - Get-PSDrive C, D | Format-Table Name, @{n="free GB"; e={[math]::Round($_.Free / 1GB)}} - New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force - Remove-Item -Recurse -Force C:\Strawberry -ErrorAction SilentlyContinue - Remove-Item -Recurse -Force "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0" -ErrorAction SilentlyContinue - - # Spotlight indexes every file the environments and builds write, using up to a - # core or two of the runner's three or four; the runner is thrown away after the job - - name: Turn off Spotlight - if: runner.os == 'macOS' - run: sudo mdutil -a -i off || true - - - name: Clean up PATH - if: runner.os == 'Windows' - uses: egor-tensin/cleanup-path@v5 - with: - dirs: 'C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin' - - - name: Disable git auto-maintenance in source caches - shell: bash -l {0} - run: git config --global maintenance.auto false - - - name: Generate recipes - shell: bash -l {0} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - for attempt in 1 2 3; do - if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi - if [[ "${attempt}" == "3" ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi - echo "Recipe generation attempt ${attempt} failed; retrying..." >&2 - sleep 15 - done - - - name: Restore build cache - uses: actions/cache/restore@v5 - with: - path: ${{ env.CACHE_DIR }} - key: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }} - restore-keys: | - ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}- - - # Temporary rebuild controls live in distros//ci.yaml. - - name: Apply ci.yaml cache controls - shell: bash -l {0} - run: | - pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" - mkdir -p "$CACHE_DIR" - pixi run rattler-index fs "$CACHE_DIR/.." --force - - - name: See packages restored by cache - shell: bash -l {0} - run: | - shopt -s nullglob - packages=("$CACHE_DIR"/*.conda) - echo "${#packages[@]} packages restored from the cache" - for package in "${packages[@]}"; do basename "$package"; done - - - name: Estimate the build time - shell: bash -l {0} - run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR" --platform "$PLATFORM" - - - name: Build recipes - id: build-recipes - # Leave time for cleanup and cache upload before the six-hour job limit. - timeout-minutes: 340 - shell: bash -l {0} - run: | - set +e - extra=() - # shorter work dirs for MAX_PATH on Windows - [ "$PLATFORM" == "win-64" ] && extra+=(--no-build-id) - # independent recipes side by side, with a shared jobserver keeping them at the - # runner's cores in total; Windows has no jobserver, and was no faster with two - [ "$PLATFORM" != "win-64" ] && extra+=(--parallel-builds "3:$(getconf _NPROCESSORS_ONLN)") - build=(python tools/robostack.py "$DISTRO" build --target-platform "$PLATFORM" "${extra[@]}") - if [ "$PLATFORM" != "win-64" ]; then - build=(python tools/ci_build.py --cache-dir "$CACHE_DIR" --timeout 19800 -- "${build[@]}") - fi - pixi run "${build[@]}" 2>&1 | tee rattler-build.log - status=${PIPESTATUS[0]} - echo "exit-code=${status}" >> "$GITHUB_OUTPUT" - exit "$status" - - - name: Summarize build result - if: always() - shell: bash -l {0} - run: | - pixi run python tools/summarize_rattler_build.py --log rattler-build.log \ - --platform "$DISTRO $PLATFORM" --outcome "${{ steps.build-recipes.outcome }}" >> "$GITHUB_STEP_SUMMARY" - - # Compare what the pull request built with the released builds, so single packages - # can be bumped without a full rebuild; it reports and never fails the build. - - name: ABI check against the released builds - id: abi-check - if: ${{ always() && matrix.target.platform == 'linux-64' && contains(fromJSON('["success","failure"]'), steps.build-recipes.outcome) }} - continue-on-error: true - shell: bash -l {0} - run: | - pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR" --summary "abi-check-$DISTRO.md" - cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" - - - name: Upload the ABI check - if: ${{ always() && steps.abi-check.outcome == 'success' }} - uses: actions/upload-artifact@v6 - with: - name: abi-check-${{ matrix.distro }} - path: abi-check-${{ matrix.distro }}.md - retention-days: 7 - - - name: Upload build log - if: always() - uses: actions/upload-artifact@v6 - with: - name: build-log-${{ matrix.distro }}-${{ matrix.target.platform }}-${{ github.run_id }}-${{ github.run_attempt }} - path: rattler-build.log - if-no-files-found: warn - retention-days: 7 - - - name: Drop the Windows package a cancellation may have cut short - # Windows builds one package at a time; POSIX builds are cleaned by ci_build.py. - if: ${{ always() && runner.os == 'Windows' && steps.build-recipes.outcome == 'cancelled' }} - shell: bash -l {0} - run: | - newest=$(ls -t "$CACHE_DIR"/*.conda 2>/dev/null | head -1) - if [ -n "$newest" ]; then echo "Removing $newest"; rm -f "$newest"; fi - - - name: Save build cache - id: save-cache - if: ${{ always() && contains(fromJSON('["success","failure","cancelled"]'), steps.build-recipes.outcome) && (runner.os == 'Windows' || steps.build-recipes.outputs.cache-ready == 'true') }} - uses: actions/cache/save@v5 - with: - path: ${{ env.CACHE_DIR }} - key: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} - - # Only the newest cache of a pull request/distribution/platform is restored; the ones - # of earlier runs and attempts only push others out of the repository's 10 GB, so - # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. - - name: Drop this pull request's superseded build caches - if: ${{ always() && steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} - continue-on-error: true - shell: bash -l {0} - env: - GH_TOKEN: ${{ github.token }} - PREFIX: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}- - KEEP: ${{ runner.os }}-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} - run: | - # cache/save reports success even on upload failure: verify the replacement exists. - if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \ - --jq "if any(.[]; .key == \"$KEEP\") then .[] | select(.key != \"$KEEP\") | .id else error(\"New cache not found\") end"); then - echo "::warning::Could not verify the new build cache; keeping all previous caches." - exit 0 - fi - while read -r id; do - [ -n "$id" ] || continue - pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" - done <<< "$ids" - - - name: Generate the staged build workflows (catches post-merge problems) - shell: bash -l {0} - run: | - for attempt in 1 2 3; do - if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi - if [[ "${attempt}" == "3" ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi - echo "Workflow generation attempt ${attempt} failed; retrying..." >&2 - sleep 15 - done + uses: ./.github/workflows/staged-build.yaml + with: + distros: ${{ needs.plan.outputs.distros }} + targets: ${{ needs.plan.outputs.targets }} # tools/robostack.py BUILD_TARGETS + upload: false # The ABI checks of all distributions as one pull-request comment, updated on every # push. Pull requests from forks get a read-only token, so the job is skipped for them: diff --git a/distros/lyrical/vinca.yaml b/distros/lyrical/vinca.yaml index 15070a04..a5ec3054 100644 --- a/distros/lyrical/vinca.yaml +++ b/distros/lyrical/vinca.yaml @@ -3,7 +3,7 @@ ros_distro: lyrical patch_dir: patch rosdistro_snapshot: rosdistro_snapshot.yaml rosdistro_additional_recipes: rosdistro_additional_recipes.yaml -build_number: 30 +build_number: 31 mutex_package: version: 0.22.0 diff --git a/pixi.lock b/pixi.lock index ab0edb4f..c6940c1e 100644 --- a/pixi.lock +++ b/pixi.lock @@ -90,7 +90,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -170,7 +170,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -243,7 +243,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/tk-8.6.13-ha6c374e_4.conda - conda: https://prefix.dev/conda-forge/osx-64/yaml-0.2.5-had63097_3.conda - conda: https://prefix.dev/conda-forge/osx-64/zstd-1.5.7-hbc1a06c_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -316,7 +316,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - conda: https://prefix.dev/conda-forge/osx-arm64/yaml-0.2.5-h74c22ad_3.conda - conda: https://prefix.dev/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -389,7 +389,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/vcomp14-14.51.36247-habf1de7_41.conda - conda: https://prefix.dev/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://prefix.dev/conda-forge/win-64/zstd-1.5.7-h534d264_7.conda - - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 + - pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl @@ -3289,7 +3289,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 size: 387535 timestamp: 1786599623274 -- pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 +- pypi: git+https://github.com/EzraBrooks/vinca.git?rev=3fa01ee61c648a062671e569c2050ed90aa07f05#3fa01ee61c648a062671e569c2050ed90aa07f05 name: vinca version: 0.2.0 requires_dist: diff --git a/pixi.toml b/pixi.toml index c76cfff9..61e3c285 100644 --- a/pixi.toml +++ b/pixi.toml @@ -36,7 +36,7 @@ git = "*" [pypi-dependencies] # One vinca for all distributions. -vinca = { git = "https://github.com/RoboStack/vinca.git", rev = "74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5" } +vinca = { git = "https://github.com/EzraBrooks/vinca.git", rev = "3fa01ee61c648a062671e569c2050ed90aa07f05" } # For local development, temporarily use (and regenerate pixi.lock): # vinca = { path = "../vinca", editable = true } diff --git a/tools/download_artifacts.cjs b/tools/download_artifacts.cjs new file mode 100644 index 00000000..21c92dde --- /dev/null +++ b/tools/download_artifacts.cjs @@ -0,0 +1,104 @@ +// Downloads artifacts of this workflow run by name, as zip files /.zip, for +// actions/github-script (staged builds, see tools/staged_build.py): +// +// await require('./tools/download_artifacts.cjs')({ names, dir, core }) +// +// download-artifact's `pattern:` filters the run's artifact list, which ends at 1,000 +// artifacts; looking each one up by name has no such limit. These are the artifact +// service's calls that @actions/artifact makes for a download by name (ListArtifacts with a +// name filter, then GetSignedArtifactURL), with the runtime token only JavaScript actions +// get. Returns the names that weren't found. +const fs = require('fs') +const path = require('path') +const { Readable } = require('stream') +const { pipeline } = require('stream/promises') + +const SERVICE = 'github.actions.results.api.v1.ArtifactService' +const PARALLEL = 8 +const ATTEMPTS = 5 +const RETRY_STATUS = [429, 500, 502, 503, 504] + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +// the run's and this job's ids in the artifact service, from the token's Actions.Results scope +function backendIds(token) { + const claims = JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString()) + for (const scope of (claims.scp || '').split(' ')) { + const [name, run, job] = scope.split(':') + if (name === 'Actions.Results' && run && job) { + return { workflow_run_backend_id: run, workflow_job_run_backend_id: job } + } + } + throw new Error('ACTIONS_RUNTIME_TOKEN has no Actions.Results scope') +} + +// request(): a fetch() response, read by use(response); network errors and the +// service's transient statuses are retried, waiting longer each time +async function retrying(what, request, use, core) { + for (let attempt = 1; ; attempt++) { + let error + let wait = 5000 * 2 ** (attempt - 1) + let response + try { + response = await request() + } catch (e) { + error = e.cause ? `${e.message} (${e.cause.message || e.cause})` : e.message + } + if (response) { + if (response.ok) { + try { + return await use(response) + } catch (e) { + error = e.message + } + } else { + error = `${response.status} ${await response.text()}` + if (!RETRY_STATUS.includes(response.status)) throw new Error(`${what}: ${error}`) + wait = Number(response.headers.get('retry-after')) * 1000 || wait + } + } + if (attempt === ATTEMPTS) throw new Error(`${what}: ${error} (after ${ATTEMPTS} attempts)`) + core.info(`${what}: ${error}; retrying in ${wait / 1000} s`) + await sleep(wait) + } +} + +module.exports = async function download({ names, dir, core }) { + const token = process.env.ACTIONS_RUNTIME_TOKEN + const results = process.env.ACTIONS_RESULTS_URL + if (!token || !results) throw new Error('ACTIONS_RUNTIME_TOKEN and ACTIONS_RESULTS_URL are only set in JavaScript actions') + const ids = backendIds(token) + const origin = new URL(results).origin + + const call = (method, body) => retrying(method, () => fetch(`${origin}/twirp/${SERVICE}/${method}`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }), (response) => response.json(), core) + + fs.mkdirSync(dir, { recursive: true }) + const missing = [] + const todo = [...new Set(names)] + const worker = async () => { + for (let name; (name = todo.shift()) !== undefined;) { + const { artifacts = [] } = await call('ListArtifacts', { ...ids, name_filter: name }) + if (!artifacts.length) { + missing.push(name) + continue + } + // the newest, as @actions/artifact takes (re-runs can upload a name again) + const artifact = artifacts.sort((a, b) => Number(b.database_id) - Number(a.database_id))[0] + const { signed_url: url } = await call('GetSignedArtifactURL', { + workflow_run_backend_id: artifact.workflow_run_backend_id, + workflow_job_run_backend_id: artifact.workflow_job_run_backend_id, + name: artifact.name, + }) + await retrying(`download ${name}`, () => fetch(url), (response) => + pipeline(Readable.fromWeb(response.body), fs.createWriteStream(path.join(dir, `${name}.zip`))), core) + core.info(`${name}: ${artifact.size} bytes`) + } + } + await Promise.all(Array.from({ length: PARALLEL }, worker)) + if (missing.length) core.info(`Not found: ${missing.sort().join(' ')}`) + return missing +} diff --git a/tools/maintenance.py b/tools/maintenance.py index 8c0f81d4..5ad765f2 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -1403,6 +1403,10 @@ def check() -> Result: text = wf.read_text() if "\non:" not in "\n" + text or "\ntrue:" in "\n" + text: problems.append(f"{wf.name}: no top-level `on:` trigger") + import staged_build + + if not staged_build.WORKFLOW.is_file() or staged_build.WORKFLOW.read_text() != staged_build.workflow(): + problems.append(f"{staged_build.WORKFLOW.relative_to(rs.ROOT)} is out of date (python tools/staged_build.py workflow)") sort = subprocess.run(["pixi", "run", "sort", "--check"], cwd=rs.ROOT, capture_output=True, text=True) if sort.returncode: problems.append("YAML files are not sorted (pixi run sort):\n" + tail(sort.stdout + sort.stderr, 10)) diff --git a/tools/robostack.py b/tools/robostack.py index 96d2f4c6..07fed69c 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -112,6 +112,8 @@ def run_tee(cmd: list[str], cwd: Path) -> tuple[int, bytes]: b"HTTP status client error (409 Conflict)", b"HTTP status server error") NETWORK_RETRIES = 4 NETWORK_WAIT = 60 # doubled on each retry: 1, 2, 4, 8 minutes +# with --continue-on-failure, rattler-build exits with 0 after a failed build or test +FAILED_MESSAGES = (b"Build failed for ", b"Test failed for ") # a build job's recipes side by side, with a shared jobserver keeping them at the runner's @@ -181,11 +183,12 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int: channel's index to catch up with the previous stage's uploads (at most INDEX_LAG_RETRIES times), or when the runner's network failed (at most NETWORK_RETRIES times, waiting longer each time). show(line) prints a line instead of - writing it as is.""" + writing it as is. A build or test that failed under --continue-on-failure fails it + too; a non-zero exit code of the command itself is printed (" exited with N").""" index_retries = network_retries = 0 while True: print("+", " ".join(cmd), f" (in {cwd.relative_to(ROOT)})", flush=True) - missing = network = False + missing = network = failed = False with subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc: for line in proc.stdout: if show: @@ -195,7 +198,11 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int: sys.stdout.buffer.flush() missing = missing or any(m in line.lower() for m in INDEX_LAG_MESSAGES) network = network or any(m in line for m in NETWORK_MESSAGES) - if proc.returncode == 0: + failed = failed or any(m in line for m in FAILED_MESSAGES) + if proc.returncode: + print(f"{Path(cmd[0]).name} exited with {proc.returncode}", file=sys.stderr, flush=True) + returncode = proc.returncode or int(failed) + if returncode == 0: return 0 if network and network_retries < NETWORK_RETRIES: wait = NETWORK_WAIT * 2 ** network_retries @@ -209,7 +216,7 @@ def run_build(cmd: list[str], cwd: Path, show=None) -> int: f"(retry {index_retries} of {INDEX_LAG_RETRIES})", file=sys.stderr, flush=True) time.sleep(INDEX_LAG_WAIT) else: - return proc.returncode + return returncode # --------------------------------------------------------------------------- # @@ -386,7 +393,10 @@ def rattler_build(distro: str, args: list[str], skip_existing: bool = True) -> l # With uploads redirected (e.g. to a test channel), later stages of the staged # build need the packages earlier stages uploaded there, and reruns skip them. extra = upload_channel_url() + # a staged build job's packages of the jobs it needs (tools/staged_build.py) + local = os.environ.get("ROBOSTACK_LOCAL_CHANNEL", "").strip() cmd = ["rattler-build", "build", "-m", "./conda_build_config.yaml", + *(["-c", (ROOT / local).resolve().as_uri()] if local else []), *(["-c", extra] if extra else []), "-c", channel_url(distro), "-c", CONDA_FORGE, "--channel-priority", "disabled"] return cmd + (["--skip-existing=all"] if skip_existing else []) + args diff --git a/tools/staged_build.py b/tools/staged_build.py new file mode 100644 index 00000000..cbdd5c98 --- /dev/null +++ b/tools/staged_build.py @@ -0,0 +1,653 @@ +"""Staged builds: vinca-gha's stages as matrix jobs, for pull requests and releases. + + python tools/staged_build.py plan --distro --platform

--os --cache-dir

--out + python tools/staged_build.py schedule --out ... + python tools/staged_build.py job --schedule --distro --platform

--job + python tools/staged_build.py artifacts --schedule --distro --platform

+ python tools/staged_build.py unpack --zips

--into + python tools/staged_build.py ready --schedule --distro --platform

--job --channel

+ python tools/staged_build.py status --schedule --distro --platform

--job --log --out

+ python tools/staged_build.py workflow [--check] + +.github/workflows/staged-build.yaml builds the given distributions and platforms in +stages, one job per batch of recipes. GitHub can't add jobs to a running workflow, so +it has a fixed number of stage slots (SLOTS) for all of them together: stage_ runs +stage k of every distribution and platform, each job as a matrix entry, after +stage_. + +- `plan` (one job per distribution and platform) lists the stages' jobs with + `vinca-gha --json`, leaving out the recipes whose every build is in the restored + pull-request cache. +- `schedule` puts the plans into the slots (at most MAX_MATRIX jobs each, GitHub's + matrix limit) and writes the schedule every stage job reads its recipes from. +- `job` is a stage job's recipes and the artifacts it needs: the packages of the jobs + that build its requirements (vinca's `needs`, and theirs), and the restored cache. + tools/download_artifacts.cjs downloads them by name (`artifacts` lists a distribution + and platform's for collecting them), `unpack` merges them into one directory. +- `ready` leaves out the job's recipes that need a recipe that wasn't built (vinca's + `requires`, in this job or the ones it needs), `status` records how each recipe went. +- `workflow` writes staged-build.yaml (`rs check` checks that it is up to date). + +Packages pass between jobs as artifacts, pkgs---, each with +status/ (success, or failure when any of its recipes wasn't built) and +status/.json ({recipe: success, failed, skipped or blocked}): a recipe that failed +holds back only the recipes that need it, and the other distributions and platforms go on. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import sys +import zipfile +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import build_timings +import robostack as rs + +SLOTS = 64 # lyrical's full linux-64 rebuild has 53 stages at --batch_size 25 +MAX_MATRIX = 256 # GitHub's limit of jobs per matrix +MAX_NAME = 60 # job names are a stage's packages, which can be many +WORKFLOW = rs.ROOT / ".github" / "workflows" / "staged-build.yaml" +PLACEHOLDER = {"distro": "-", "platform": "-", "os": "ubuntu-latest", "job": "-", "name": "-"} + + +def artifact(distro: str, platform: str, job: str) -> str: + return f"pkgs-{distro}-{platform}-{job}" + + +def cached_recipes(recipes: Path, platform: str, cache_dir: Path) -> set[str]: + """The recipes whose every build (one per variant) is in the cache (/*.conda).""" + names = {p.parent.name for p in recipes.glob("*/recipe.yaml")} + variants = build_timings.variant_builds(recipes, platform) if names else {} + cached = Counter(f.name.rsplit("-", 2)[0] for f in cache_dir.glob("*/*.conda")) + return {r for r in names if cached[r] >= variants.get(r, 1)} + + +def fold(stages: list[list[dict]], slots: int = SLOTS) -> list[list[dict]]: + """The stages as at most `slots` stages: the ones past the last slot become one job + there (build-ci orders a job's recipes by their dependencies).""" + if len(stages) <= slots: + return stages + rest = [job for stage in stages[slots - 1:] for job in stage] + ids = {job["id"] for job in rest} + tail = { + "id": f"stage_{slots - 1}_rest", + "name": f"{len(rest)} jobs of stages {slots - 1}+", + "recipes": " ".join(job["recipes"] for job in rest), + "needs": sorted({n for job in rest for n in job["needs"]} - ids), + "requires": {r: deps for job in rest for r, deps in job.get("requires", {}).items()}, + } + return [*stages[:slots - 1], [tail]] + + +def plan(distro: str, platform: str, cache_dir: Path, batch_size: int) -> list[list[dict]]: + w = rs.work_dir(distro) + recipes = w / "recipes" + cached = cached_recipes(recipes, platform, cache_dir) if recipes.is_dir() else set() + # vinca-gha stages what is in ./recipes; the cached recipes count as built, like the + # ones on the channel that generate-recipes -n leaves out + aside = w / "recipes-cached" + shutil.rmtree(aside, ignore_errors=True) + aside.mkdir() + for r in cached: + (recipes / r).rename(aside / r) + try: + out = w / "stages.json" + cmd = ["vinca-gha", "--platform", platform, "-d", "./recipes", "--batch_size", str(batch_size), + "--json", out.name] + if build_timings.vinca_timings(distro, platform, w / "build_timings.json"): + cmd += ["--timings", "build_timings.json"] + if rs.run(cmd, w): + raise SystemExit("vinca-gha failed") + stages = json.loads(out.read_text())["stages"] + finally: + for r in cached: + (aside / r).rename(recipes / r) + aside.rmdir() + print(f"{len(cached)} recipes fully in the cache; {sum(map(len, stages))} jobs in {len(stages)} stages") + return fold(stages) + + +def merge_smallest(stages: list[list[dict]], k: int) -> None: + """Merge the two jobs of stage k with the fewest recipes into one.""" + a, b = sorted(stages[k], key=lambda job: len(job["recipes"].split()))[:2] + a["recipes"] = f"{a['recipes']} {b['recipes']}" + a["name"] = f"{a['name']} {b['name']}" + a["needs"] = sorted(set(a["needs"]) | set(b["needs"])) + a["requires"] = {**a.get("requires", {}), **b.get("requires", {})} + stages[k].remove(b) + for stage in stages[k + 1:]: + for job in stage: + if b["id"] in job["needs"]: + job["needs"] = sorted((set(job["needs"]) - {b["id"]}) | {a["id"]}) + + +def schedule(plans: list[dict], slots: int = SLOTS, max_matrix: int = MAX_MATRIX) -> tuple[list[list[dict]], dict]: + """The slots' matrix entries, and each distribution and platform's jobs by id + ({"/": {id: {"recipes", "needs", "requires"}}}). A slot with more + than max_matrix jobs has the smallest jobs of its largest plans merged.""" + entries = [] + for k in range(slots): + while sum(len(p["stages"][k]) for p in plans if k < len(p["stages"])) > max_matrix: + largest = max((p for p in plans if k < len(p["stages"])), key=lambda p: len(p["stages"][k])) + merge_smallest(largest["stages"], k) + entries.append([ + {"distro": p["distro"], "platform": p["platform"], "os": p["os"], "job": job["id"], + "name": short(f"{p['distro']} {p['platform']} {job['name']}")} + for p in plans if k < len(p["stages"]) for job in p["stages"][k] + ]) + while entries and not entries[-1]: + entries.pop() + jobs = {f"{p['distro']}/{p['platform']}": {job["id"]: {"recipes": job["recipes"], "needs": job["needs"], + "requires": job.get("requires", {})} + for stage in p["stages"] for job in stage} + for p in plans} + return entries, jobs + + +def short(name: str) -> str: + return name if len(name) <= MAX_NAME else name[:MAX_NAME - 1] + "…" + + +def job_needs(jobs: dict[str, dict], job: str) -> list[str]: + """The jobs whose packages the job needs: its needs, and theirs.""" + seen: set[str] = set() + todo = list(jobs[job]["needs"]) + while todo: + if (j := todo.pop()) not in seen: + seen.add(j) + todo += jobs[j]["needs"] + return sorted(seen) + + +def unpack(zips: Path, into: Path) -> None: + """Extract the downloaded artifacts (tools/download_artifacts.cjs) into one directory.""" + into.mkdir(parents=True, exist_ok=True) + for z in sorted(zips.glob("*.zip")): + with zipfile.ZipFile(z) as f: + f.extractall(into) + z.unlink() + + +def needed_status(jobs: dict[str, dict], needed: list[str], channel: Path) -> dict[str, str]: + """How each recipe of the needed jobs went, from their status/.json; a job + without one (its artifact missing: it didn't get to the end) built none.""" + status = {} + for j in needed: + f = channel / "status" / f"{j}.json" + recipes = json.loads(f.read_text()) if f.is_file() else {} + for r in jobs[j]["recipes"].split(): + status[r] = recipes.get(r, "not built") + return status + + +def blocked(job: dict, status: dict[str, str]) -> dict[str, str]: + """The job's recipes that can't be built, each with the requirement it lacks: one + that the needed jobs didn't build, or one of the job's own recipes held back.""" + out: dict[str, str] = {} + changed = True + while changed: + changed = False + for r in job["recipes"].split(): + if r in out: + continue + lacking = [q for q in job["requires"].get(r, []) if q in out or status.get(q, "success") != "success"] + if lacking: + out[r] = lacking[0] + changed = True + return out + + +def outcomes(job: dict, held: dict[str, str], log: str | None) -> dict[str, str]: + """How each recipe of the job went: blocked (held back), else from the last attempt in + rattler-build's log (rs build-ci with --continue-on-failure=skip-dependents): failed + (its build or test), skipped (it needs one that failed in this job), or success. When + rattler-build itself failed (or there's no log), every recipe that isn't blocked failed. + rattler-build skips only the outputs that need a failed one to build, so one that only + runs with it (e.g. ros--, of ros2-) counts as skipped here.""" + recipes = job["recipes"].split() + attempt = re.split(r"^\+ rattler-build ", log, flags=re.M)[-1] if log else "" + aborted = not attempt or re.search(r"^rattler-build exited with ", attempt, re.M) + name = lambda identifier: identifier.rsplit("-", 2)[0] # noqa: E731 (name-version-build) + failed = {name(i) for i in re.findall(r"(?:Build|Test) failed for (\S+): ", attempt)} + skipped = {name(i) for i in re.findall(r"Skipping (\S+) because its dependency \S+ failed to build", attempt)} + out = {} + for r in recipes: + if r in held: + out[r] = "blocked" + elif r in failed or aborted: + out[r] = "failed" + elif r in skipped: + out[r] = "skipped" + else: + out[r] = "success" + changed = True + while changed: + changed = False + for r in recipes: + if out[r] == "success" and any(out.get(q, "success") != "success" for q in job["requires"].get(r, [])): + out[r] = "skipped" + changed = True + return out + + +def workflow() -> str: + """staged-build.yaml: HEADER, the stage slots, then COLLECT.""" + secrets = json.dumps({d: rs.anaconda_token_secret(d) for d in rs.distros()}, separators=(",", ":")) + slots = [] + for k in range(SLOTS): + needs = "schedule" if k == 0 else f"[schedule, stage_{k - 1}]" + slots.append(SLOT.replace("@K@", str(k)).replace("@NEEDS@", needs).replace("@SECRETS@", secrets)) + outputs = "".join(f" slot_{k}: ${{{{ steps.schedule.outputs.slot_{k} }}}}\n" for k in range(SLOTS)) + every = ", ".join(["schedule", *(f"stage_{k}" for k in range(SLOTS))]) + return HEADER.replace("@OUTPUTS@", outputs) + "".join(slots) + COLLECT.replace("@NEEDS@", every) + + +def write_lines(env: str, lines: list[str]) -> None: + if path := os.environ.get(env): + with open(path, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("plan", help="the stages of one distribution and platform") + p.add_argument("--distro", required=True) + p.add_argument("--platform", required=True) + p.add_argument("--os", required=True, help="the platform's runner") + p.add_argument("--cache-dir", required=True, type=Path) + p.add_argument("--batch-size", type=int, default=25) + p.add_argument("--out", required=True, type=Path) + s = sub.add_parser("schedule", help="the plans in the stage slots, as GitHub outputs") + s.add_argument("--out", required=True, type=Path) + s.add_argument("plans", nargs="*", type=Path) + j = sub.add_parser("job", help="a stage job's RECIPES, NEEDED and NEEDED_ARTIFACTS, as GitHub environment") + a = sub.add_parser("artifacts", help="a distribution and platform's artifacts, ARTIFACTS, as GitHub environment") + r = sub.add_parser("ready", help="the job's RECIPES whose requirements were built, as GitHub environment") + t = sub.add_parser("status", help="how each of the job's recipes went, status/.json in --out") + for p in (j, a, r, t): + p.add_argument("--schedule", required=True, type=Path) + p.add_argument("--distro", required=True) + p.add_argument("--platform", required=True) + for p in (j, r, t): + p.add_argument("--job", required=True) + r.add_argument("--channel", required=True, type=Path, help="the needed jobs' artifacts, unpacked") + t.add_argument("--log", required=True, type=Path, help="the build's output (rs build-ci)") + t.add_argument("--out", required=True, type=Path, help="the job's artifact") + u = sub.add_parser("unpack", help="extract downloaded artifacts into one directory") + u.add_argument("--zips", required=True, type=Path) + u.add_argument("--into", required=True, type=Path) + w = sub.add_parser("workflow", help=f"write {WORKFLOW.relative_to(rs.ROOT)}") + w.add_argument("--check", action="store_true", help="only check that it is up to date") + args = parser.parse_args() + + if args.command == "workflow": + text = workflow() + if args.check: + if not WORKFLOW.is_file() or WORKFLOW.read_text() != text: + print(f"{WORKFLOW.relative_to(rs.ROOT)} is out of date (python tools/staged_build.py workflow)") + return 1 + return 0 + WORKFLOW.write_text(text) + return 0 + + if args.command == "plan": + stages = plan(args.distro, args.platform, args.cache_dir.resolve(), args.batch_size) + doc = {"distro": args.distro, "platform": args.platform, "os": args.os, "stages": stages} + args.out.write_text(json.dumps(doc) + "\n") + jobs = sum(map(len, stages)) + write_lines("GITHUB_STEP_SUMMARY", [f"**{args.distro} {args.platform}**: {jobs} build jobs in {len(stages)} stages", ""]) + for k, stage in enumerate(stages): + print(f"stage {k}: " + "; ".join(job["recipes"] for job in stage)) + return 0 + + if args.command == "schedule": + plans = [json.loads(f.read_text()) for f in sorted(args.plans)] + entries, jobs = schedule(plans) + args.out.write_text(json.dumps(jobs) + "\n") + # the slots past the last stage are skipped, but GitHub reads their matrix anyway + lines = [f"count={len(entries)}", + "combos=" + json.dumps([{"distro": p["distro"], "platform": p["platform"]} for p in plans])] + lines += [f"slot_{k}=" + json.dumps(entries[k] if k < len(entries) else [PLACEHOLDER], separators=(",", ":")) + for k in range(SLOTS)] + write_lines("GITHUB_OUTPUT", lines) + print(f"{sum(map(len, entries))} jobs in {len(entries)} stages: " + ", ".join(str(len(e)) for e in entries)) + return 0 + + if args.command == "unpack": + unpack(args.zips, args.into) + return 0 + + jobs = json.loads(args.schedule.read_text())[f"{args.distro}/{args.platform}"] + if args.command == "artifacts": + names = [artifact(args.distro, args.platform, j) for j in ["cache", *jobs]] + write_lines("GITHUB_ENV", [f"ARTIFACTS={' '.join(names)}"]) + print(f"{len(names)} artifacts") + return 0 + + job = jobs[args.job] + needed = job_needs(jobs, args.job) + held_file = Path(f"blocked-{args.job}.json") + if args.command == "job": + lines = [f"RECIPES={job['recipes']}", f"NEEDED={' '.join(needed)}", + "NEEDED_ARTIFACTS=" + " ".join(artifact(args.distro, args.platform, j) for j in ["cache", *needed])] + write_lines("GITHUB_ENV", lines) + print("\n".join(lines)) + return 0 + + if args.command == "ready": + status = needed_status(jobs, needed, args.channel) + held = blocked(job, status) + held_file.write_text(json.dumps(held) + "\n") + for r, q in held.items(): + print(f"{r}: not built, as it needs {q} ({status.get(q, 'held back in this job')})") + ready = [r for r in job["recipes"].split() if r not in held] + write_lines("GITHUB_ENV", [f"RECIPES={' '.join(ready)}"]) + print(f"{len(ready)} of {len(job['recipes'].split())} recipes to build: {' '.join(ready)}") + return 0 + + held = json.loads(held_file.read_text()) if held_file.is_file() else {} + log = args.log.read_text(encoding="utf-8", errors="replace") if args.log.is_file() else None + result = outcomes(job, held, log) + (args.out / "status").mkdir(parents=True, exist_ok=True) + (args.out / "status" / f"{args.job}.json").write_text(json.dumps(result, indent=1) + "\n") + ok = all(s == "success" for s in result.values()) + (args.out / "status" / args.job).write_text(("success" if ok else "failure") + "\n") + lines = [f"| {r} | {s}{f' (needs {held[r]})' if r in held else ''} |" for r, s in result.items() if s != "success"] + for line in lines: + print(line.strip("| ").replace(" | ", ": ")) + if lines: + write_lines("GITHUB_STEP_SUMMARY", ["| recipe | not built |", "|---|---|", *lines, ""]) + # the packages of the recipes that built, to upload (releases) + built = {r for r, s in result.items() if s == "success"} + Path("uploads.txt").write_text("".join(f"{p.as_posix()}\n" for p in sorted(args.out.glob("*/*.conda")) + if p.name.rsplit("-", 2)[0] in built)) + print(f"{len(built)} of {len(result)} recipes built") + return 0 if ok else 1 + + +SETUP = """\ + - uses: actions/checkout@v6 + with: + persist-credentials: false + # git auto-maintenance can leave a lock file in the git clones pixi caches + # (e.g. vinca's), which breaks every later run restoring that cache + - name: Disable git auto-maintenance + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - uses: prefix-dev/setup-pixi@v0.11.0 + with: + locked: true # the lock file must match pixi.toml + pixi-version: v0.81.0 + cache-write: false # saved from main only (pixi-cache.yaml) +""" + +HEADER = """\ +# Generated by tools/staged_build.py (python tools/staged_build.py workflow); `pixi run rs check` +# checks that it is up to date. +# +# The given distributions and platforms, built in vinca-gha's stages, one job per batch +# of recipes (stage-job.yaml): pull requests (testpr.yaml) keep the packages as the +# pull request's build cache, releases (release.yaml) upload them. See +# tools/staged_build.py for how the stages become jobs. +name: Staged build + +on: + workflow_call: + inputs: + distros: + description: JSON list of distributions + required: true + type: string + targets: + description: 'JSON list of platforms and their runners, [{"os": ..., "platform": ...}]' + required: true + type: string + upload: + description: Upload the packages to the distributions' channels (else keep them as the pull request's cache) + required: true + type: boolean + +env: + PYTHONIOENCODING: utf-8 + # while uploads go to a test channel, packages already there aren't built again + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} + # the pull request's packages, /*.conda; restored and saved on Linux for every platform + CACHE_DIR: pr-cache + +jobs: + plan: + strategy: + fail-fast: false + matrix: + distro: ${{ fromJSON(inputs.distros) }} + target: ${{ fromJSON(inputs.targets) }} + name: plan ${{ matrix.distro }} ${{ matrix.target.platform }} + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.target.platform }} + steps: +""" + SETUP + """\ + - name: Generate recipes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -e + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi + echo "Recipe generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done + # a tarball: artifacts drop the build scripts' executable bit + - name: Pack the recipes + run: mkdir -p "distros/$DISTRO/work/recipes" && tar -cf recipes.tar -C "distros/$DISTRO/work" recipes + - name: Upload the recipes + uses: actions/upload-artifact@v6 + with: + name: recipes-${{ matrix.distro }}-${{ matrix.target.platform }} + path: recipes.tar + overwrite: true + retention-days: 7 + - name: Restore build cache + if: ${{ !inputs.upload }} + uses: actions/cache/restore@v5 + with: + path: ${{ env.CACHE_DIR }} + key: staged-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }} + restore-keys: | + staged-${{ matrix.distro }}-${{ matrix.target.platform }}-pr-${{ github.event.pull_request.number }}- + # Temporary rebuild controls live in distros//ci.yaml. + - name: Apply ci.yaml cache controls + run: | + for subdir in "$PLATFORM" noarch; do + mkdir -p "$CACHE_DIR/$subdir" + pixi run python tools/ci_config.py --cache-dir "$CACHE_DIR/$subdir" --config "distros/$DISTRO/ci.yaml" --vinca "distros/$DISTRO/vinca.yaml" + done + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages restored from the cache" + for package in "${packages[@]}"; do basename "$package"; done + mkdir -p "$CACHE_DIR/status" && echo success > "$CACHE_DIR/status/cache" + - name: Upload the cached packages + uses: actions/upload-artifact@v6 + with: + name: pkgs-${{ matrix.distro }}-${{ matrix.target.platform }}-cache + path: ${{ env.CACHE_DIR }} + compression-level: 0 # .conda files are compressed + overwrite: true + retention-days: 3 + - name: Estimate the build time + run: pixi run python tools/ci_estimate.py --recipes "distros/$DISTRO/work/recipes" --cache-dir "$CACHE_DIR/$PLATFORM" --platform "$PLATFORM" + - name: Plan the stages + env: + OS: ${{ matrix.target.os }} + run: pixi run python tools/staged_build.py plan --distro "$DISTRO" --platform "$PLATFORM" --os "$OS" --cache-dir "$CACHE_DIR" --out "plan-$DISTRO-$PLATFORM.json" + - uses: actions/upload-artifact@v6 + with: + name: plan-${{ matrix.distro }}-${{ matrix.target.platform }} + path: plan-${{ matrix.distro }}-${{ matrix.target.platform }}.json + overwrite: true + retention-days: 7 + + # Every plan in the stage slots, and the schedule the stage jobs read their recipes from; + # a failed plan leaves out only its distribution and platform. + schedule: + needs: plan + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + outputs: + count: ${{ steps.schedule.outputs.count }} + combos: ${{ steps.schedule.outputs.combos }} +@OUTPUTS@ steps: +""" + SETUP + """\ + - uses: actions/download-artifact@v7 + with: + pattern: plan-* + merge-multiple: true + path: plans + - name: Schedule the stages + id: schedule + run: pixi run python tools/staged_build.py schedule --out schedule.json plans/*.json + - uses: actions/upload-artifact@v6 + with: + name: schedule + path: schedule.json + overwrite: true + retention-days: 7 + +""" + +# A slot runs after the one before even when that one failed: the jobs check that +# their own requirements built (stage-job.yaml), so the other distributions and +# platforms go on. +SLOT = """\ + stage_@K@: + needs: @NEEDS@ + if: ${{ !cancelled() && needs.schedule.result == 'success' && fromJSON(needs.schedule.outputs.count) > @K@ }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.slot_@K@) }} + name: @K@ ${{ matrix.name }} + uses: ./.github/workflows/stage-job.yaml + secrets: inherit + with: + distro: ${{ matrix.distro }} + platform: ${{ matrix.platform }} + os: ${{ matrix.os }} + job: ${{ matrix.job }} + upload: ${{ inputs.upload }} + token_secret: ${{ fromJSON('@SECRETS@')[matrix.distro] }} + +""" + +COLLECT = """\ + # Pull requests: the packages of every stage as the pull request's build cache (a new + # push builds only what changed), then the ABI check of what the pull request built. + collect: + needs: [@NEEDS@] + if: ${{ always() && !inputs.upload && needs.schedule.result == 'success' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.schedule.outputs.combos) }} + name: collect ${{ matrix.distro }} ${{ matrix.platform }} + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distro }} + PLATFORM: ${{ matrix.platform }} + steps: +""" + SETUP + """\ + - uses: actions/download-artifact@v7 + with: + name: schedule + - name: The stage jobs' artifacts + run: pixi run python tools/staged_build.py artifacts --schedule schedule.json --distro "$DISTRO" --platform "$PLATFORM" + # by name: download-artifact's pattern only sees the run's first 1,000 artifacts + - name: Download the stage jobs' packages + uses: actions/github-script@v8 + with: + script: | + await require('./tools/download_artifacts.cjs')({ names: process.env.ARTIFACTS.split(' '), dir: 'artifacts', core }) + - name: See the packages + run: | + pixi run python tools/staged_build.py unpack --zips artifacts --into "$CACHE_DIR" + rm -rf "$CACHE_DIR/status" + shopt -s nullglob + packages=("$CACHE_DIR"/*/*.conda) + echo "${#packages[@]} packages" + for package in "${packages[@]}"; do basename "$package"; done + - name: Save build cache + id: save-cache + uses: actions/cache/save@v5 + with: + path: ${{ env.CACHE_DIR }} + key: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + + # Only the newest cache of a pull request/distribution/platform is restored; the ones + # of earlier runs and attempts only push others out of the repository's 10 GB, so + # full rebuilds lost their progress. Forks have a read-only token, so skip deletion. + - name: Drop this pull request's superseded build caches + if: ${{ steps.save-cache.outcome == 'success' && github.event.pull_request.number && github.event.pull_request.head.repo.full_name == github.repository }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PREFIX: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}- + KEEP: staged-${{ matrix.distro }}-${{ matrix.platform }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }} + run: | + # cache/save reports success even on upload failure: verify the replacement exists. + if ! ids=$(pixi run gh cache list --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF" --key "$PREFIX" --limit 100 --json id,key \\ + --jq "if any(.[]; .key == \\"$KEEP\\") then .[] | select(.key != \\"$KEEP\\") | .id else error(\\"New cache not found\\") end"); then + echo "::warning::Could not verify the new build cache; keeping all previous caches." + exit 0 + fi + while read -r id; do + [ -n "$id" ] || continue + pixi run gh cache delete "$id" --repo "$GITHUB_REPOSITORY" && echo "deleted cache $id" + done <<< "$ids" + + # Compare what the pull request built with the released builds, so single packages + # can be bumped without a full rebuild; it reports and never fails the build. + - name: ABI check against the released builds + id: abi-check + if: matrix.platform == 'linux-64' + continue-on-error: true + run: | + pixi run abi-check --distro "$DISTRO" --platform "$PLATFORM" --pr-builds "$CACHE_DIR/$PLATFORM" --summary "abi-check-$DISTRO.md" + cat "abi-check-$DISTRO.md" >> "$GITHUB_STEP_SUMMARY" + - name: Upload the ABI check + if: ${{ steps.abi-check.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: abi-check-${{ matrix.distro }} + path: abi-check-${{ matrix.distro }}.md + retention-days: 7 + + # the build branches' workflows, while they still build the releases + - uses: actions/download-artifact@v7 + with: + name: recipes-${{ matrix.distro }}-${{ matrix.platform }} + - name: Prepare the work directory + run: pixi run rs "$DISTRO" prepare && tar -xf recipes.tar -C "distros/$DISTRO/work" + - name: Generate the staged build workflows (catches post-merge problems) + run: | + if [ -z "$(ls "distros/$DISTRO/work/recipes" 2>/dev/null)" ]; then echo "Nothing to build"; exit 0; fi + for attempt in 1 2 3; do + if pixi run rs "$DISTRO" gha --platform "$PLATFORM" --batch-size 25; then break; fi + if [[ "$attempt" == 3 ]]; then echo "Workflow generation failed after 3 attempts" >&2; exit 1; fi + echo "Workflow generation attempt $attempt failed; retrying..." >&2 + sleep 15 + done +""" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/test_ci_build.py b/tools/tests/test_ci_build.py index 3c62f29b..1b3acc8a 100644 --- a/tools/tests/test_ci_build.py +++ b/tools/tests/test_ci_build.py @@ -80,8 +80,8 @@ def test_interruption_stops_descendants_and_preserves_only_finished_archives(tmp @pytest.mark.parametrize("lookup_status,ids,deleted", [(0, "11\n22", "11\n22\n"), (1, "11", ""), (0, "", "")]) def test_cache_pruning_requires_verified_replacement(tmp_path, lookup_status, ids, deleted): - workflow = yaml.safe_load((ROOT / ".github/workflows/testpr.yaml").read_text()) - prune = next(s for s in workflow["jobs"]["build"]["steps"] + workflow = yaml.safe_load((ROOT / ".github/workflows/staged-build.yaml").read_text()) + prune = next(s for s in workflow["jobs"]["collect"]["steps"] if s.get("name") == "Drop this pull request's superseded build caches") pixi = tmp_path / "pixi" pixi.write_text('''#!/bin/bash diff --git a/tools/tests/test_staged_build.py b/tools/tests/test_staged_build.py new file mode 100644 index 00000000..d6d82fdc --- /dev/null +++ b/tools/tests/test_staged_build.py @@ -0,0 +1,184 @@ +"""tools/staged_build.py: the staged build's plans, schedule and stage slots.""" + +import sys + +import yaml + +import robostack as rs +import staged_build as sb + + +def job(id: str, recipes: str, needs=()) -> dict: + return {"id": id, "name": recipes, "recipes": recipes, "needs": list(needs)} + + +def plan(distro: str, platform: str, stages: list[list[dict]]) -> dict: + return {"distro": distro, "platform": platform, "os": "ubuntu-latest", "stages": stages} + + +def test_fold_puts_the_stages_past_the_last_slot_into_one_job(): + stages = [[job("s0", "ros2-a")], [job("s1", "ros2-b", ["s0"])], + [job("s2a", "ros2-c", ["s1"]), job("s2b", "ros2-d", ["s0"])], [job("s3", "ros2-e", ["s2a"])]] + + folded = sb.fold(stages, slots=3) + + assert len(folded) == 3 + (tail,) = folded[2] + assert tail["recipes"] == "ros2-c ros2-d ros2-e" + assert tail["needs"] == ["s0", "s1"] + + +def test_cached_recipes_need_every_variant_in_the_cache(monkeypatch, tmp_path): + recipes, cache = tmp_path / "recipes", tmp_path / "cache" + for name in ("ros2-a", "ros2-b", "ros2-c"): + (recipes / name).mkdir(parents=True) + (recipes / name / "recipe.yaml").write_text("{}\n") + (cache / "linux-64").mkdir(parents=True) + (cache / "noarch").mkdir() + (cache / "linux-64" / "ros2-a-1.0.0-py312_0.conda").touch() + (cache / "linux-64" / "ros2-a-1.0.0-py313_0.conda").touch() + (cache / "linux-64" / "ros2-b-1.0.0-py312_0.conda").touch() + (cache / "noarch" / "ros2-c-1.0.0-0.conda").touch() + monkeypatch.setattr(sb.build_timings, "variant_builds", lambda path, platform: {"ros2-a": 2, "ros2-b": 2}) + + assert sb.cached_recipes(recipes, "linux-64", cache) == {"ros2-a", "ros2-c"} + + +def test_schedule_shares_the_slots_between_distributions_and_platforms(): + plans = [plan("jazzy", "linux-64", [[job("stage_0_job_0", "ros2-a")], [job("stage_1_job_0", "ros2-b")]]), + plan("lyrical", "win-64", [[job("stage_0_job_0", "ros2-a"), job("stage_0_job_1", "ros2-c")]])] + + entries, jobs = sb.schedule(plans) + + assert [[(e["distro"], e["platform"], e["job"]) for e in slot] for slot in entries] == [ + [("jazzy", "linux-64", "stage_0_job_0"), ("lyrical", "win-64", "stage_0_job_0"), + ("lyrical", "win-64", "stage_0_job_1")], + [("jazzy", "linux-64", "stage_1_job_0")], + ] + assert jobs["lyrical/win-64"]["stage_0_job_1"] == {"recipes": "ros2-c", "needs": [], "requires": {}} + + +def test_schedule_merges_jobs_past_the_matrix_limit(): + stages = [[job("a", "ros2-a"), job("b", "ros2-b ros2-bb"), job("c", "ros2-c")], [job("d", "ros2-d", ["c"])]] + stages[0][2]["requires"] = {"ros2-c": []} + stages[1][0]["requires"] = {"ros2-d": ["ros2-c"]} + + entries, jobs = sb.schedule([plan("jazzy", "linux-64", stages)], max_matrix=2) + + assert [e["job"] for e in entries[0]] == ["a", "b"] + assert jobs["jazzy/linux-64"]["a"]["recipes"] == "ros2-a ros2-c" + assert jobs["jazzy/linux-64"]["a"]["requires"] == {"ros2-c": []} + assert jobs["jazzy/linux-64"]["d"]["needs"] == ["a"] + + +def test_a_job_needs_its_requirements_jobs_and_theirs(): + jobs = {"a": {"needs": []}, "b": {"needs": ["a"]}, "c": {"needs": []}, "d": {"needs": ["b"]}} + + assert sb.job_needs(jobs, "d") == ["a", "b"] + + +def test_a_failed_recipe_blocks_only_the_recipes_that_need_it(tmp_path): + jobs = { + "s0": {"recipes": "ros2-zstd ros2-mola", "needs": [], "requires": {}}, + "s1": {"recipes": "ros2-a ros2-b ros2-c ros2-d", "needs": ["s0"], + "requires": {"ros2-a": ["ros2-zstd"], "ros2-b": ["ros2-mola"], "ros2-c": ["ros2-b"], "ros2-d": ["ros2-x"]}}, + "s2": {"recipes": "ros2-x", "needs": [], "requires": {}}, + } + (tmp_path / "status").mkdir() + (tmp_path / "status" / "s0.json").write_text('{"ros2-zstd": "success", "ros2-mola": "failed"}') + + # s2's artifact is missing: it didn't finish + status = sb.needed_status(jobs, ["s0", "s2"], tmp_path) + + assert status == {"ros2-zstd": "success", "ros2-mola": "failed", "ros2-x": "not built"} + # ros2-c needs ros2-b, held back in the same job + assert sb.blocked(jobs["s1"], status) == {"ros2-b": "ros2-mola", "ros2-c": "ros2-b", "ros2-d": "ros2-x"} + + +LOG = """\ ++ rattler-build build --recipe ./recipes/ros2-a (in distros/lyrical/work) +10:00:00 [ros2-a] Build failed for ros2-a-1.0.0-np2py312h0_31: network +rattler-build exited with 1 +Network error; retrying in 60 s (retry 1 of 4) ++ rattler-build build --recipe ./recipes/ros2-a (in distros/lyrical/work) +10:01:00 [ros2-a] Archive written to 'output/linux-64/ros2-a-1.0.0-np2py312h0_31.conda' +10:02:00 [ros2-mola] Build failed for ros2-mola-3.3.0-np2py312hdd89193_31: Script failed to execute +10:02:00 [-] Skipping ros-lyrical-mola-3.3.0-h9490d1a_31 because its dependency ros2-mola failed to build +10:03:00 [ros2-t] Test failed for ros2-t-1.0.0-h0_31: import failed +""" + + +def test_each_recipe_s_outcome_from_the_last_attempt(): + job = {"recipes": "ros2-a ros2-mola ros-lyrical-mola ros2-t ros-lyrical-t ros2-held", "requires": {"ros-lyrical-t": ["ros2-t"]}} + + # ros-lyrical-t only runs with ros2-t: rattler-build built it + assert sb.outcomes(job, {"ros2-held": "ros2-x"}, LOG) == { + "ros2-a": "success", "ros2-mola": "failed", "ros-lyrical-mola": "skipped", "ros2-t": "failed", + "ros-lyrical-t": "skipped", "ros2-held": "blocked", + } + + +def test_every_recipe_failed_when_rattler_build_itself_did(): + log = "+ rattler-build build\n10:00:00 [ros2-a] Archive written to 'x'\nrattler-build exited with 1\n" + job = {"recipes": "ros2-a ros2-b", "requires": {}} + + assert sb.outcomes(job, {}, log) == {"ros2-a": "failed", "ros2-b": "failed"} + assert sb.outcomes(job, {"ros2-b": "ros2-x"}, None) == {"ros2-a": "failed", "ros2-b": "blocked"} + + +def test_unpack_merges_the_artifacts(tmp_path): + import zipfile + + zips = tmp_path / "zips" + zips.mkdir() + for name, files in {"pkgs-a": ["linux-64/ros2-a-1-0.conda", "status/a"], "pkgs-b": ["noarch/ros2-b-1-0.conda", "status/b"]}.items(): + with zipfile.ZipFile(zips / f"{name}.zip", "w") as z: + for f in files: + z.writestr(f, "x") + + sb.unpack(zips, tmp_path / "channel") + + assert sorted(p.relative_to(tmp_path / "channel").as_posix() for p in (tmp_path / "channel").rglob("*") if p.is_file()) == [ + "linux-64/ros2-a-1-0.conda", "noarch/ros2-b-1-0.conda", "status/a", "status/b"] + assert not list(zips.iterdir()) + + +def test_a_build_that_failed_under_continue_on_failure_fails(monkeypatch, tmp_path, capsys): + script = "import sys; print('{\"fields\": {\"message\": \"Build failed for ros2-a-1-0: x\"}}')" + monkeypatch.setattr(rs, "ROOT", tmp_path) + + assert rs.run_build([sys.executable, "-c", script], tmp_path) == 1 + assert rs.run_build([sys.executable, "-c", "print('ok')"], tmp_path) == 0 + assert rs.run_build([sys.executable, "-c", "raise SystemExit(3)"], tmp_path) == 3 + assert "exited with 3" in capsys.readouterr().err + + +def test_every_slot_runs_after_the_one_before(): + jobs = yaml.safe_load(sb.workflow())["jobs"] + + assert jobs["stage_0"]["needs"] == "schedule" + for k in range(1, sb.SLOTS): + assert jobs[f"stage_{k}"]["needs"] == ["schedule", f"stage_{k - 1}"] + assert f"> {k} }}" in jobs[f"stage_{k}"]["if"] + assert jobs["collect"]["needs"] == ["schedule", *(f"stage_{k}" for k in range(sb.SLOTS))] + assert set(jobs["schedule"]["outputs"]) == {"count", "combos", *(f"slot_{k}" for k in range(sb.SLOTS))} + + +def test_the_workflow_is_up_to_date(): + assert sb.WORKFLOW.read_text() == sb.workflow(), "python tools/staged_build.py workflow" + + +def test_rattler_build_reads_the_local_channel(monkeypatch): + monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False) + monkeypatch.setenv("ROBOSTACK_LOCAL_CHANNEL", "pr-channel") + + cmd = rs.rattler_build("jazzy", []) + + assert cmd[cmd.index("-c") + 1] == (rs.ROOT / "pr-channel").as_uri() + + +def test_rattler_build_without_the_local_channel(monkeypatch): + monkeypatch.delenv("ROBOSTACK_LOCAL_CHANNEL", raising=False) + monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False) + + assert not any(c.startswith("file:") for c in rs.rattler_build("jazzy", []))