Generate recipes, snapshots and build workflows with vinca-rs

.claude/skills/robostack-debug-build/SKILL.md

@@ -63,11 +63,11 @@ bash -x conda_build.sh 2>&1 | less
   endif()
   ```
 - **Windows `min`/`max` macro clashes**: `target_compile_definitions(<target> PRIVATE NOMINMAX WIN32_LEAN_AND_MEAN)` under `if(WIN32)`.
-- **gtest / test failures**: add the missing dependency in `dependencies.yaml`, or
+- **gtest / test failures**: add the missing dependency in `pkg_additional_info.yaml`, or
   disable the tests when safe; no custom shims.
 - **Qt plugins (rviz, rtabmap)**: make sure CMake finds the intended Qt major version.
-- **Missing dependency**: `shared/patch/dependencies.yaml` (all distributions) or
-  `distros/<d>/patch/dependencies.yaml` (one), keyed by ROS package name:
+- **Missing dependency**: `shared/pkg_additional_info.yaml` (all distributions) or
+  `distros/<d>/pkg_additional_info.yaml` (one), keyed by ROS package name:
   ```yaml
   <ros_pkg>:
     add_host: [<conda-pkg>]

.claude/skills/robostack-patch/SKILL.md

@@ -13,8 +13,9 @@ description: Create, refresh, place, port and validate source patches for RoboSt
   of adding variants.
 - vinca picks them up by name when generating recipes; nothing else needs wiring.
   `pixi run rs check` rejects other names.
-- `distros/<d>/patch/dependencies.yaml` is for dependency fixes, not source patches
-  (`shared/patch/dependencies.yaml` for all distributions).
+- Dependency fixes are not source patches: they are `add_*`/`remove_*` keys in
+  `distros/<d>/pkg_additional_info.yaml` (`shared/pkg_additional_info.yaml` for all
+  distributions).
 
 ## Create a patch from work-directory edits
 

.github/ISSUE_TEMPLATE/3_robostack_bot.yaml

@@ -20,14 +20,15 @@ body:
         | `dependency-report` | maintainers | Check the dependencies against the pins and update the distribution's dependency-report issue (closed when nothing conflicts). Runs weekly. |
         | `update-build-timings` | maintainers | Record the build times of the last week's build-branch runs in `shared/build_timings/`, which the build workflows use to pack each stage's recipes into jobs by time. Runs weekly. |
         | `update-pixi` | maintainers | Upgrade the tooling's dependencies in `pixi.toml` / `pixi.lock` to their newest versions (`pixi upgrade`; not Python or the capped ones) and check. The PR lists the changes. Runs monthly. |
+        | `sync-pixi-mappings` | maintainers | Copy the rosdep mappings (`robostack.yaml`) to pixi-build-ros and open or update a PR on prefix-dev/pixi. Runs daily. |
 
         On a pull request, `@robostack-bot rebuild-dependents` (maintainers) commits the rebuilds its ABI check suggests to the pull request's branch.
   - type: dropdown
     id: command
     attributes:
       label: Command
       options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild, update-vinca,
-                dependency-report, update-build-timings, update-pixi]
+                dependency-report, update-build-timings, update-pixi, sync-pixi-mappings]
     validations:
       required: true
   - type: input

.github/actions/build-recipes/action.yaml

@@ -0,0 +1,76 @@
+# Build and upload some of a distribution's recipes: the steps of every job of the
+# build workflows (.github/build-template.yaml). One action keeps each generated job
+# a few lines long, so many more jobs fit GitHub's 500 KiB workflow file limit.
+name: Build recipes
+description: Build the given recipes of distros/<distro>/work/recipes and upload them
+inputs:
+  recipes:
+    description: Space-separated recipe names
+    required: true
+  platform:
+    description: Target platform (conda subdir)
+    required: true
+  token:
+    description: anaconda.org API token of the distribution's organization
+    required: false
+  keep-file:
+    description: File kept as an artifact (the build times the job recorded)
+    required: false
+runs:
+  using: composite
+  steps:
+    # git auto-maintenance can leave a lock file in the git clones pixi caches,
+    # which breaks every later run restoring that cache
+    - name: Disable git auto-maintenance
+      shell: bash
+      run: git config --global maintenance.auto false && git config --global gc.auto 0
+    - name: Setup pixi
+      uses: prefix-dev/setup-pixi@v0.11.0
+      with:
+        pixi-version: v0.81.0
+        cache: "true"
+        log-level: v
+        frozen: "true"
+    - if: runner.os == 'Windows'
+      uses: egor-tensin/cleanup-path@v5
+      with:
+        dirs: C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin
+    - if: runner.os == 'Windows'
+      name: conda-forge build setup
+      shell: cmd
+      run: |
+        :: hosted runners may have libcrypto/libssl DLLs in System32, found before
+        :: the environment's openssl
+        DEL C:\Windows\System32\libcrypto-1_1-x64.dll || (Echo Ignoring failure to delete libcrypto)
+        DEL C:\Windows\System32\libssl-1_1-x64.dll || (Echo Ignoring failure to delete libssl)
+    - if: runner.os == 'Windows'
+      name: Build
+      shell: cmd
+      env:
+        ANACONDA_API_TOKEN: ${{ inputs.token }}
+        CURRENT_RECIPES: ${{ inputs.recipes }}
+        CONDA_BLD_PATH: C:\\bld\\
+        VINCA_CUSTOM_CMAKE_BUILD_DIR: C:\\x\\
+        PYTHONUNBUFFERED: "1"
+      run: call .scripts\build_win.bat
+    - if: runner.os != 'Windows'
+      name: Build
+      shell: bash
+      env:
+        ANACONDA_API_TOKEN: ${{ inputs.token }}
+        CURRENT_RECIPES: ${{ inputs.recipes }}
+        BUILD_TARGET: ${{ inputs.platform }}
+        PYTHONUNBUFFERED: "1"
+        CI: github
+      run: |
+        export GIT_BRANCH=${GITHUB_REF_NAME}
+        export FEEDSTOCK_NAME=${GITHUB_REPOSITORY##*/}
+        .scripts/build_unix.sh --target "$BUILD_TARGET"
+    - name: Keep build times
+      if: always() && inputs.keep-file != ''
+      uses: actions/upload-artifact@v6
+      with:
+        name: ${{ github.job }}-${{ github.run_attempt }}
+        path: ${{ inputs.keep-file }}
+        if-no-files-found: ignore
+        retention-days: 30

.github/build-template.yaml

@@ -0,0 +1,48 @@
+# Template of the build workflows (`pixi run rs <distro> gha`, vinca-rs workflow).
+# Not a workflow itself: vinca-rs copies jobs.build once per job, sets each copy's
+# name and needs (exactly the jobs building its recipes' dependencies) and replaces
+# @recipes@ with its recipes; @distro@, @platform@, @branch@, @secret@ and
+# @keep_file@ come from tools/robostack.py. Selectors (if/then/else) are resolved
+# for the platform. The steps are .github/actions/build-recipes.
+name: Build branch
+run-name: "@distro@ @platform@"
+on:
+  push:
+    branches: ["@branch@"]
+# one run per distribution and platform at a time: a newer run (the latest main's
+# recipes and workflow) cancels the older one, and skips what that one uploaded
+# already. Re-run failed jobs only in the newest run.
+concurrency:
+  group: build-@distro@-@platform@
+  cancel-in-progress: true
+env:
+  ROBOSTACK_DISTRO: "@distro@"
+  # optional repository variable: upload to this prefix.dev channel instead
+  # (e.g. a test channel), or 'none' to only build
+  ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }}
+jobs:
+  build:
+    runs-on:
+      if: linux and x86_64
+      then: ubuntu-latest
+      else:
+        if: linux
+        then: ubuntu-24.04-arm
+        else:
+          if: osx and x86_64
+          then: macos-15-intel
+          else:
+            if: osx
+            then: macos-15
+            else: windows-2022
+    # jobs that declare permissions get none they don't list; prefix.dev uploads use
+    # trusted publishing (id-token), attestations are published too
+    permissions: {contents: read, id-token: write, attestations: write}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/build-recipes
+        with:
+          recipes: "@recipes@"
+          platform: "@platform@"
+          token: ${{ secrets.@secret@ }}
+          keep-file: "@keep_file@"

.github/workflows/bot.yaml

@@ -11,13 +11,18 @@
 #   nothing conflicts; and a PR with the build times the week's build-branch runs
 #   recorded (shared/build_timings/, for packing the build jobs by time).
 # - monthly (the 1st): a PR upgrading the tooling's pixi dependencies (pixi.toml / pixi.lock).
+# - daily, when vars.PIXI_FORK is set: a PR on prefix-dev/pixi copying the rosdep mappings
+#   (shared/ and distros/*/robostack.yaml) to pixi-build-ros, so that `pixi build` of ROS
+#   packages maps rosdep keys as RoboStack does. It is pushed to the fork vars.PIXI_FORK
+#   (<owner>/pixi) with secrets.PIXI_SYNC_TOKEN, a token of that fork's owner: the bot's
+#   GitHub App can't open PRs on prefix-dev/pixi.
 # - on demand (owners, members and collaborators):
 #     Actions > robostack-bot > Run workflow, an issue from the "robostack-bot command"
 #     template, or a comment: @robostack-bot <command> [<distro>]
 #   commands: update-rosdistro-snapshot [<distro>...|all] | find-stale-packages [<distro>...|all] |
 #             update-conda-forge-pinning [<distro>...|all] | add-package <pkg>... [<distro>...] |
 #             rebuild <pkg>... [<distro>...|all] [--with-dependents] | update-vinca [<ref>] |
-#             dependency-report [<distro>...|all] | update-build-timings | update-pixi |
+#             dependency-report [<distro>...|all] | update-build-timings | update-pixi | sync-pixi-mappings |
 #             rebuild-dependents (a comment on a pull request: commits the rebuilds its ABI
 #             check suggests to its branch; not possible for pull requests from forks)
 # - package requests (the "Package request" issue form or `@robostack-bot add-package`,
@@ -38,7 +43,7 @@ on:
       command:
         type: choice
         options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild,
-                  update-vinca, dependency-report, update-build-timings, update-pixi]
+                  update-vinca, dependency-report, update-build-timings, update-pixi, sync-pixi-mappings]
       distro:
         description: Distribution (for the per-distribution commands; empty = all)
         required: false
@@ -89,6 +94,7 @@ jobs:
           ASSOCIATION: ${{ github.event.comment.author_association || github.event.issue.author_association }}
           PULL_REQUEST: ${{ github.event.issue.pull_request && github.event.issue.number || '' }}
           SCHEDULE: ${{ github.event.schedule }}
+          PIXI_FORK: ${{ vars.PIXI_FORK }}
         run: |
           case "$EVENT" in
             schedule)
@@ -104,8 +110,9 @@ jobs:
                 exit 0
               fi
               # rosdistro syncs, and the conda-forge pinning: one job for all distributions
-              # (they share it; one with its own pinning is updated in the same pull request)
-              jobs=$(pixi run -q rs rosdistro-syncs | python3 -c 'import json,sys; print(json.dumps(json.load(sys.stdin) + [{"command":"update-conda-forge-pinning","distro":"","args":"","preview":False}]))') ;;
+              # (they share it; one with its own pinning is updated in the same pull request);
+              # the rosdep mappings for pixi-build-ros, once a pixi fork is set up
+              jobs=$(pixi run -q rs rosdistro-syncs | python3 -c 'import json,os,sys; print(json.dumps(json.load(sys.stdin) + [{"command":"update-conda-forge-pinning","distro":"","args":"","preview":False}] + ([{"command":"sync-pixi-mappings","distro":"","args":"","preview":False}] if os.environ.get("PIXI_FORK") else [])))') ;;
             workflow_dispatch)  # only people with write access can dispatch
               jobs=$(ROBOSTACK_BOT_BODY="@robostack-bot $COMMAND $PACKAGES $DISTRO" ROBOSTACK_BOT_ASSOCIATION=OWNER pixi run -q rs parse-command) ;;
             *)
@@ -146,6 +153,13 @@ jobs:
           locked: true  # the lock file must match pixi.toml
           pixi-version: v0.81.0
           cache-write: false  # saved from main only (pixi-cache.yaml)
+      - name: Check out prefix-dev/pixi
+        if: matrix.command == 'sync-pixi-mappings'
+        uses: actions/checkout@v6
+        with:
+          repository: prefix-dev/pixi
+          path: _pixi
+          persist-credentials: false
       - name: robostack-bot ${{ matrix.command }} ${{ matrix.distro }}
         id: bot
         env:
@@ -178,6 +192,8 @@ jobs:
               # shellcheck disable=SC2046  # the --exclude arguments, one word each
               pixi upgrade --json $(pixi run -q rs update-pixi --excludes) > "$RUNNER_TEMP/pixi-upgrade.json" &&
                 pixi run rs update-pixi --changes "$RUNNER_TEMP/pixi-upgrade.json" --summary "$RUNNER_TEMP/summary.md" ;;
+            # writes pixi-build-ros's copies of the mappings into the pixi checkout
+            sync-pixi-mappings) pixi run rs sync-pixi-mappings _pixi --summary "$RUNNER_TEMP/summary.md" ;;
             # creates / updates the distribution's dependency-report issue
             dependency-report) pixi run rs "$DISTRO" dependency-report --summary "$RUNNER_TEMP/summary.md" ;;
           esac
@@ -192,7 +208,9 @@ jobs:
           printf '\nCloses #%s\n' "$NUMBER" >> "$RUNNER_TEMP/summary.md"
       - name: Open or update the pull request
         id: pr
-        if: steps.bot.outputs.changed == 'true' && !matrix.preview && matrix.command != 'rebuild-dependents'
+        if: >-
+          steps.bot.outputs.changed == 'true' && !matrix.preview &&
+          matrix.command != 'rebuild-dependents' && matrix.command != 'sync-pixi-mappings'
         uses: peter-evans/create-pull-request@v8
         with:
           token: ${{ steps.app.outputs.token || secrets.GHA_PAT || github.token }}
@@ -206,14 +224,27 @@ jobs:
           labels: |
             robostack-bot
             ${{ steps.bot.outputs.labels }}
+      - name: Open or update the pull request on prefix-dev/pixi
+        id: pixi_pr
+        if: steps.bot.outputs.changed == 'true' && matrix.command == 'sync-pixi-mappings'
+        uses: peter-evans/create-pull-request@v8
+        with:
+          path: _pixi
+          token: ${{ secrets.PIXI_SYNC_TOKEN }}
+          push-to-fork: ${{ vars.PIXI_FORK }}
+          branch: robostack-bot/sync-mappings
+          delete-branch: true
+          commit-message: "chore(pixi-build-ros): update RoboStack's rosdep mappings"
+          title: "chore(pixi-build-ros): update RoboStack's rosdep mappings"
+          body-path: ${{ runner.temp }}/summary.md
       - name: Reply on the issue / pull request
         if: always() && (github.event_name == 'issue_comment' || github.event_name == 'issues')
         env:
           GH_TOKEN: ${{ github.token }}
           NUMBER: ${{ github.event.issue.number }}
           TITLE: ${{ steps.bot.outputs.title || 'failed' }}
-          PR_URL: ${{ steps.pr.outputs.pull-request-url }}
-          PR_OPERATION: ${{ steps.pr.outputs.pull-request-operation }}
+          PR_URL: ${{ steps.pr.outputs.pull-request-url || steps.pixi_pr.outputs.pull-request-url }}
+          PR_OPERATION: ${{ steps.pr.outputs.pull-request-operation || steps.pixi_pr.outputs.pull-request-operation }}
         run: |
           {
             echo "**robostack-bot ${{ matrix.command }} ${{ matrix.distro }}**: $TITLE"

.github/workflows/testpr.yaml

@@ -1,6 +1,6 @@
 # Pull-request build: only the distributions a PR touches are built
 # (a change outside distros/ affects all of them), on all five platforms.
-name: Build
+name: Pull request
 
 on:
   pull_request:
@@ -10,8 +10,9 @@ on:
         description: Space-separated distributions to build (default all)
         required: false
 
+# a fixed key, not github.workflow: renaming the workflow mustn't let two runs overlap
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: pr-build-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 permissions:
@@ -24,6 +25,7 @@ env:
 
 jobs:
   plan:
+    name: "plan: distros and platforms to build"
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -67,6 +69,7 @@ jobs:
           echo "targets=$(pixi run python tools/robostack.py build-targets --labels "$labels")" | tee -a "$GITHUB_OUTPUT"
 
   check:
+    name: "checks: tooling tests, generated files"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -97,11 +100,11 @@ jobs:
       fail-fast: false
       matrix:
         distro: ${{ fromJSON(needs.plan.outputs.distros) }}
-    name: ${{ matrix.distro }} patches
+    name: "${{ matrix.distro }}: patches apply"
     runs-on: ubuntu-latest
     env:
       DISTRO: ${{ matrix.distro }}
-      # generate-recipes -n leaves out what is on the channel, like the builds do
+      # generate-recipes leaves out what is on the channel (same build string), like the builds do
       ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }}
     steps:
       - uses: actions/checkout@v6
@@ -123,7 +126,7 @@ jobs:
           all=$(mktemp -d)
           for platform in linux-64 linux-aarch64 osx-64 osx-arm64 win-64; do
             for attempt in 1 2 3; do
-              if pixi run rs "$DISTRO" generate-recipes --platform "$platform" -n; then break; fi
+              if pixi run rs "$DISTRO" generate-recipes --platform "$platform"; then break; fi
               if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed for $platform" >&2; exit 1; fi
               sleep 15
             done
@@ -147,7 +150,7 @@ jobs:
       matrix:
         distro: ${{ fromJSON(needs.plan.outputs.distros) }}
         target: ${{ fromJSON(needs.plan.outputs.targets) }}  # tools/robostack.py BUILD_TARGETS
-    name: ${{ matrix.distro }} ${{ matrix.target.platform }}
+    name: "${{ matrix.distro }} ${{ matrix.target.platform }}: build"
     runs-on: ${{ matrix.target.os }}
     env:
       DISTRO: ${{ matrix.distro }}
@@ -217,7 +220,7 @@ jobs:
       run: |
         set -e
         for attempt in 1 2 3; do
-          if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi
+          if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM"; then break; fi
           if [[ "${attempt}" == "3" ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi
           echo "Recipe generation attempt ${attempt} failed; retrying..." >&2
           sleep 15

.scripts/build_unix.sh

@@ -40,14 +40,16 @@ pixi run -v rs "${ROBOSTACK_DISTRO}" build-ci "${recipe_args[@]}" --output-dir $
 
 # Check if it build something, this is a hotfix for the skips inside additional_recipes
 shopt -s nullglob
-conda_packages=( "${CONDA_BLD_PATH}/${target}"*/*.conda )
+# noarch packages (the ros-<distro>-* compatibility packages) are built and uploaded
+# by every platform's jobs: the first upload wins, the others are skipped
+conda_packages=( "${CONDA_BLD_PATH}/${target}"*/*.conda "${CONDA_BLD_PATH}/noarch/"*.conda )
 if (( ${#conda_packages[@]} > 0 )); then
     # Upload packages one-by-one; the upload task (tools/robostack.py) uploads them to
     # prefix.dev and anaconda.org and skips packages that are already there.
     for conda_package in "${conda_packages[@]}"; do
         pixi run rs "${ROBOSTACK_DISTRO}" upload "${conda_package}"
     done
 else
-    echo "Warning: No .conda files found in ${CONDA_BLD_PATH}/${target}"
+    echo "Warning: No .conda files found in ${CONDA_BLD_PATH}/${target} or ${CONDA_BLD_PATH}/noarch"
     echo "This might be due to all the packages being skipped"
 fi