diff --git a/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml b/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml index c4e10102..14653408 100644 --- a/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml +++ b/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml @@ -20,6 +20,7 @@ body: | `dependency-report` | maintainers | Check the dependencies against the pins and update the distribution's dependency-report issue (closed when nothing conflicts). Runs weekly. | | `update-build-timings` | maintainers | Record the build times of the last week's build-branch runs in `shared/build_timings/`, which the build workflows use to pack each stage's recipes into jobs by time. Runs weekly. | | `update-pixi` | maintainers | Upgrade the tooling's dependencies in `pixi.toml` / `pixi.lock` to their newest versions (`pixi upgrade`; not Python or the capped ones) and check. The PR lists the changes. Runs monthly. | + | `sync-pixi-mappings` | maintainers | Copy the rosdep mappings (`robostack.yaml`) to pixi-build-ros and open or update a PR on prefix-dev/pixi. Runs daily. | On a pull request, `@robostack-bot rebuild-dependents` (maintainers) commits the rebuilds its ABI check suggests to the pull request's branch. - type: dropdown @@ -27,7 +28,7 @@ body: attributes: label: Command options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild, update-vinca, - dependency-report, update-build-timings, update-pixi] + dependency-report, update-build-timings, update-pixi, sync-pixi-mappings] validations: required: true - type: input diff --git a/.github/workflows/bot.yaml b/.github/workflows/bot.yaml index 56a49236..896664ab 100644 --- a/.github/workflows/bot.yaml +++ b/.github/workflows/bot.yaml @@ -11,13 +11,18 @@ # nothing conflicts; and a PR with the build times the week's build-branch runs # recorded (shared/build_timings/, for packing the build jobs by time). # - monthly (the 1st): a PR upgrading the tooling's pixi dependencies (pixi.toml / pixi.lock). +# - daily, when vars.PIXI_FORK is set: a PR on prefix-dev/pixi copying the rosdep mappings +# (shared/ and distros/*/robostack.yaml) to pixi-build-ros, so that `pixi build` of ROS +# packages maps rosdep keys as RoboStack does. It is pushed to the fork vars.PIXI_FORK +# (/pixi) with secrets.PIXI_SYNC_TOKEN, a token of that fork's owner: the bot's +# GitHub App can't open PRs on prefix-dev/pixi. # - on demand (owners, members and collaborators): # Actions > robostack-bot > Run workflow, an issue from the "robostack-bot command" # template, or a comment: @robostack-bot [] # commands: update-rosdistro-snapshot [...|all] | find-stale-packages [...|all] | # update-conda-forge-pinning [...|all] | add-package ... [...] | # rebuild ... [...|all] [--with-dependents] | update-vinca [] | -# dependency-report [...|all] | update-build-timings | update-pixi | +# dependency-report [...|all] | update-build-timings | update-pixi | sync-pixi-mappings | # rebuild-dependents (a comment on a pull request: commits the rebuilds its ABI # check suggests to its branch; not possible for pull requests from forks) # - package requests (the "Package request" issue form or `@robostack-bot add-package`, @@ -38,7 +43,7 @@ on: command: type: choice options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild, - update-vinca, dependency-report, update-build-timings, update-pixi] + update-vinca, dependency-report, update-build-timings, update-pixi, sync-pixi-mappings] distro: description: Distribution (for the per-distribution commands; empty = all) required: false @@ -89,6 +94,7 @@ jobs: ASSOCIATION: ${{ github.event.comment.author_association || github.event.issue.author_association }} PULL_REQUEST: ${{ github.event.issue.pull_request && github.event.issue.number || '' }} SCHEDULE: ${{ github.event.schedule }} + PIXI_FORK: ${{ vars.PIXI_FORK }} run: | case "$EVENT" in schedule) @@ -104,8 +110,9 @@ jobs: exit 0 fi # rosdistro syncs, and the conda-forge pinning: one job for all distributions - # (they share it; one with its own pinning is updated in the same pull request) - jobs=$(pixi run -q rs rosdistro-syncs | python3 -c 'import json,sys; print(json.dumps(json.load(sys.stdin) + [{"command":"update-conda-forge-pinning","distro":"","args":"","preview":False}]))') ;; + # (they share it; one with its own pinning is updated in the same pull request); + # the rosdep mappings for pixi-build-ros, once a pixi fork is set up + jobs=$(pixi run -q rs rosdistro-syncs | python3 -c 'import json,os,sys; print(json.dumps(json.load(sys.stdin) + [{"command":"update-conda-forge-pinning","distro":"","args":"","preview":False}] + ([{"command":"sync-pixi-mappings","distro":"","args":"","preview":False}] if os.environ.get("PIXI_FORK") else [])))') ;; workflow_dispatch) # only people with write access can dispatch jobs=$(ROBOSTACK_BOT_BODY="@robostack-bot $COMMAND $PACKAGES $DISTRO" ROBOSTACK_BOT_ASSOCIATION=OWNER pixi run -q rs parse-command) ;; *) @@ -146,6 +153,13 @@ jobs: locked: true # the lock file must match pixi.toml pixi-version: v0.81.0 cache-write: false # saved from main only (pixi-cache.yaml) + - name: Check out prefix-dev/pixi + if: matrix.command == 'sync-pixi-mappings' + uses: actions/checkout@v6 + with: + repository: prefix-dev/pixi + path: _pixi + persist-credentials: false - name: robostack-bot ${{ matrix.command }} ${{ matrix.distro }} id: bot env: @@ -178,6 +192,8 @@ jobs: # shellcheck disable=SC2046 # the --exclude arguments, one word each pixi upgrade --json $(pixi run -q rs update-pixi --excludes) > "$RUNNER_TEMP/pixi-upgrade.json" && pixi run rs update-pixi --changes "$RUNNER_TEMP/pixi-upgrade.json" --summary "$RUNNER_TEMP/summary.md" ;; + # writes pixi-build-ros's copies of the mappings into the pixi checkout + sync-pixi-mappings) pixi run rs sync-pixi-mappings _pixi --summary "$RUNNER_TEMP/summary.md" ;; # creates / updates the distribution's dependency-report issue dependency-report) pixi run rs "$DISTRO" dependency-report --summary "$RUNNER_TEMP/summary.md" ;; esac @@ -192,7 +208,9 @@ jobs: printf '\nCloses #%s\n' "$NUMBER" >> "$RUNNER_TEMP/summary.md" - name: Open or update the pull request id: pr - if: steps.bot.outputs.changed == 'true' && !matrix.preview && matrix.command != 'rebuild-dependents' + if: >- + steps.bot.outputs.changed == 'true' && !matrix.preview && + matrix.command != 'rebuild-dependents' && matrix.command != 'sync-pixi-mappings' uses: peter-evans/create-pull-request@v8 with: token: ${{ steps.app.outputs.token || secrets.GHA_PAT || github.token }} @@ -206,14 +224,27 @@ jobs: labels: | robostack-bot ${{ steps.bot.outputs.labels }} + - name: Open or update the pull request on prefix-dev/pixi + id: pixi_pr + if: steps.bot.outputs.changed == 'true' && matrix.command == 'sync-pixi-mappings' + uses: peter-evans/create-pull-request@v8 + with: + path: _pixi + token: ${{ secrets.PIXI_SYNC_TOKEN }} + push-to-fork: ${{ vars.PIXI_FORK }} + branch: robostack-bot/sync-mappings + delete-branch: true + commit-message: "chore(pixi-build-ros): update RoboStack's rosdep mappings" + title: "chore(pixi-build-ros): update RoboStack's rosdep mappings" + body-path: ${{ runner.temp }}/summary.md - name: Reply on the issue / pull request if: always() && (github.event_name == 'issue_comment' || github.event_name == 'issues') env: GH_TOKEN: ${{ github.token }} NUMBER: ${{ github.event.issue.number }} TITLE: ${{ steps.bot.outputs.title || 'failed' }} - PR_URL: ${{ steps.pr.outputs.pull-request-url }} - PR_OPERATION: ${{ steps.pr.outputs.pull-request-operation }} + PR_URL: ${{ steps.pr.outputs.pull-request-url || steps.pixi_pr.outputs.pull-request-url }} + PR_OPERATION: ${{ steps.pr.outputs.pull-request-operation || steps.pixi_pr.outputs.pull-request-operation }} run: | { echo "**robostack-bot ${{ matrix.command }} ${{ matrix.distro }}**: $TITLE" diff --git a/README.md b/README.md index 9792dcbd..a2f0a016 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ shared/ the same for every distribution pkg_additional_info.yaml per-package settings that are the same everywhere patch/dependencies.yaml dependency fixes that are the same everywhere robostack.yaml rosdep key -> conda package mapping (`robostack: []` drops a key) + (also copied to pixi-build-ros: robostack-bot sync-pixi-mappings) pinning/ conda-forge pinning version, migrations and overrides tests/ package tests (ros2-.yaml; `if: ros_distro ...` for distro-specific bits) tools/robostack.py `pixi run rs ...`: every task, for one distribution or the repository @@ -152,6 +153,7 @@ waiting for the macOS runners; remove them for a full build before merging. | `dependency-report [... \| all]` | maintainers | Runs the dependency check (`check-deps`, linux-64) and keeps one issue per distribution, *``: dependency report* (label `dependency-report`), up to date with the conflicts, notes and conda-forge migration status; the issue is closed when nothing conflicts. Runs weekly (Mondays). | | `update-build-timings` | maintainers | Merges the build times the last week's build-branch runs recorded (`rs build-ci`, kept as each job's artifact) into `shared/build_timings/.json` and opens a PR. The build workflows pack each stage's recipes into jobs by these times. Runs weekly (Mondays). Labelled `build:none` (see below). | | `update-pixi` | maintainers | Runs `pixi upgrade`: the newest versions of the tooling's dependencies in `pixi.toml` and `pixi.lock` (rattler-build, rattler-index, gh, pytest, ...), except Python (moved by hand) and requirements that are only an upper bound (`cmake = "<4.0"`: capped on purpose); the vinca pin is `update-vinca`'s. Runs `rs check`; the PR lists the changes as a table (pixi-diff-to-markdown). Runs monthly (on the 1st). Labelled `build:none` (see below). | +| `sync-pixi-mappings` | maintainers | Copies the rosdep mappings (`shared/robostack.yaml`, and each `distros//robostack.yaml` over it) to pixi-build-ros (`crates/pixi_build_ros/robostack.yaml` and `robostack_distros.yaml` in prefix-dev/pixi) and opens or updates a PR there, listing the added, removed and changed keys: `pixi build` of ROS packages then maps rosdep keys as RoboStack does. Mappings are only changed here. Runs daily once `vars.PIXI_FORK` (the fork the PR is pushed to) and `secrets.PIXI_SYNC_TOKEN` (a token of that fork's owner) are set. | The PRs of `update-vinca`, `update-build-timings` and `update-pixi` don't change packages directly, but touch files outside `distros/`, which would build every distribution: they are labelled `build:none`, so only the checks run (and nothing is built while a build is under way). To build one, remove the label, or replace it with `build:` labels. diff --git a/distros/humble/pkg_additional_info.yaml b/distros/humble/pkg_additional_info.yaml index 360be2e8..89ae6964 100644 --- a/distros/humble/pkg_additional_info.yaml +++ b/distros/humble/pkg_additional_info.yaml @@ -111,6 +111,8 @@ ament_cmake_gtest: # uses conda-forge's gtest instead of gtest_vendor's sources build_number: 23 aruco_ros: # its recipe test stops the node for sure (package tests hung on macOS) build_number: 23 +plotjuggler: + build_number: 24 rtabmap_slam: # its recipe test stops the node for sure (package tests hung on macOS) build_number: 23 spatio_temporal_voxel_layer: diff --git a/distros/jazzy/pkg_additional_info.yaml b/distros/jazzy/pkg_additional_info.yaml index 129677d4..8dee9d07 100644 --- a/distros/jazzy/pkg_additional_info.yaml +++ b/distros/jazzy/pkg_additional_info.yaml @@ -149,6 +149,8 @@ nav2_rviz_plugins: build_number: 28 octomap_rviz_plugins: build_number: 28 +plotjuggler: + build_number: 30 rtabmap_slam: # its recipe test stops the node for sure (package tests hung on macOS) build_number: 28 rviz_2d_overlay_plugins: diff --git a/distros/kilted/robostack.yaml b/distros/kilted/robostack.yaml new file mode 100644 index 00000000..69b32c96 --- /dev/null +++ b/distros/kilted/robostack.yaml @@ -0,0 +1,72 @@ +# kilted: its own rosdep mappings, over shared/robostack.yaml: +# Qt 5 (kilted's rviz and rqt need Qt5 / PyQt5), and the xtensor its packages were built +# with. kilted (EOL) isn't built here (no vinca.yaml); this is for pixi-build-ros +# (robostack-bot sync-pixi-mappings). +libqt5-core: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-gui: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-opengl: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-opengl-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-svg: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-svg-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-websockets-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-widgets: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5x11extras5-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +python-qt5-bindings: + robostack: [pyqt, pyqt-builder] +python3-qt5-bindings: + robostack: [pyqt, pyqt-builder] +python3-qt5-bindings-gl: + robostack: [pyqt, pyqt-builder] +qt5-image-formats-plugins: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +qt5-qmake: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +qtbase5-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +xtensor: + robostack: [xtensor ==0.24.7] diff --git a/distros/lyrical/pkg_additional_info.yaml b/distros/lyrical/pkg_additional_info.yaml index 5266ede7..a3e043f3 100644 --- a/distros/lyrical/pkg_additional_info.yaml +++ b/distros/lyrical/pkg_additional_info.yaml @@ -127,12 +127,18 @@ gz_sim_vendor: build_number: 33 gz_transport_vendor: build_number: 33 +launch: + build_number: 34 moveit_ros_visualization: build_number: 31 moveit_task_constructor_visualization: build_number: 31 +nav2_common: + build_number: 34 nav2_rviz_plugins: build_number: 31 +plotjuggler: + build_number: 34 rtabmap_rviz_plugins: build_number: 31 rtabmap_slam: # its recipe test stops the node for sure (package tests hung on macOS) diff --git a/distros/noetic/robostack.yaml b/distros/noetic/robostack.yaml new file mode 100644 index 00000000..f8b7e757 --- /dev/null +++ b/distros/noetic/robostack.yaml @@ -0,0 +1,71 @@ +# noetic: its own rosdep mappings, over shared/robostack.yaml: +# Qt 5, and the xtensor its packages were built with. noetic isn't built here (no +# vinca.yaml); this is for pixi-build-ros (robostack-bot sync-pixi-mappings). +libqt5-core: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-gui: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-opengl: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-opengl-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-svg: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-svg-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-websockets-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5-widgets: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +libqt5x11extras5-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +python-qt5-bindings: + robostack: [pyqt, pyqt-builder] +python3-qt5-bindings: + robostack: [pyqt, pyqt-builder] +python3-qt5-bindings-gl: + robostack: [pyqt, pyqt-builder] +qt5-image-formats-plugins: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +qt5-qmake: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +qtbase5-dev: + robostack: + linux: [qt-main, libopengl-devel, libgl-devel] + osx: [qt-main] + win64: [qt-main] +xtensor: + robostack: [xtensor ==0.24.7] diff --git a/distros/rolling/pkg_additional_info.yaml b/distros/rolling/pkg_additional_info.yaml index 8e3d8466..cd6bec2e 100644 --- a/distros/rolling/pkg_additional_info.yaml +++ b/distros/rolling/pkg_additional_info.yaml @@ -75,10 +75,18 @@ aruco_ros: # its recipe test stops the node for sure (package tests hung on mac build_number: 33 fuse_viz: build_number: 33 +launch: + build_number: 35 moveit_ros_visualization: build_number: 33 moveit_task_constructor_visualization: build_number: 33 +nav2_common: + build_number: 35 +orbbec_camera: + build_number: 35 +plotjuggler: + build_number: 35 rviz_2d_overlay_plugins: build_number: 33 rviz_common: diff --git a/pixi.toml b/pixi.toml index 7b2eaaf9..7accaeb1 100644 --- a/pixi.toml +++ b/pixi.toml @@ -44,7 +44,7 @@ vinca = { git = "https://github.com/RoboStack/vinca.git", rev = "74d1e856d5b00c0 [tasks] rs = { cmd = "python tools/robostack.py", description = "Run a task for one distribution: `pixi run rs ` (generate-recipes, build, build-one, check-patches, check-deps, create-snapshot, render-pinning, sort, ...)." } -sort = { cmd = "sh -lc 'vinca-sort-vinca-lists $@ shared/vinca.yaml && vinca-sort-yaml-keys $@ shared/robostack.yaml shared/pkg_additional_info.yaml shared/patch/dependencies.yaml && for d in distros/*/; do (cd $d && vinca-sort-vinca-lists $@ vinca.yaml && vinca-sort-yaml-keys $@ pkg_additional_info.yaml rosdistro_additional_recipes.yaml $(ls patch/dependencies.yaml 2>/dev/null)) || exit 1; done' --", description = "Sort the YAML files of all distributions (`--check` to only check)." } +sort = { cmd = "sh -lc 'vinca-sort-vinca-lists $@ shared/vinca.yaml && vinca-sort-yaml-keys $@ shared/robostack.yaml shared/pkg_additional_info.yaml shared/patch/dependencies.yaml && for d in distros/*/; do [ -f $d/vinca.yaml ] || continue; (cd $d && vinca-sort-vinca-lists $@ vinca.yaml && vinca-sort-yaml-keys $@ pkg_additional_info.yaml rosdistro_additional_recipes.yaml $(ls patch/dependencies.yaml 2>/dev/null)) || exit 1; done' --", description = "Sort the YAML files of all distributions (`--check` to only check)." } test = { cmd = "python -m pytest tools/tests", description = "Unit tests of the repository tooling (tools/tests)." } abi-check = { cmd = "pixi exec -s python=3.14 -s zstandard -s pyyaml -s pyelftools python tools/abi_check.py", description = "Compare the ABI of package builds with libabigail (Linux only): `pixi run abi-check --distro jazzy --latest-pairs 30`, `--old a.conda --new b.conda`, or `--pr-builds ` (a pull request's builds against the released ones)." } -render-pinning = { cmd = "sh -lc 'for d in distros/*/; do python tools/robostack.py $(basename $d) render-pinning || exit 1; done'", description = "Regenerate every distribution's conda_build_config.yaml from shared/pinning and its distro.yaml." } +render-pinning = { cmd = "sh -lc 'for d in distros/*/; do [ -f $d/vinca.yaml ] || continue; python tools/robostack.py $(basename $d) render-pinning || exit 1; done'", description = "Regenerate every distribution's conda_build_config.yaml from shared/pinning and its distro.yaml." } diff --git a/shared/robostack.yaml b/shared/robostack.yaml index 3a1510ae..246b3203 100644 --- a/shared/robostack.yaml +++ b/shared/robostack.yaml @@ -81,6 +81,8 @@ cppunit: win64: [] cppzmq: robostack: [cppzmq] +crypto++: + robostack: [cryptopp] curl: robostack: [libcurl] curlpp-dev: @@ -211,6 +213,8 @@ ignition-transport11: robostack: [libignition-transport11] ignition-transport8: robostack: [libignition-transport8] +iproute2: + robostack: [] ipython3: robostack: [ipython] java: @@ -717,6 +721,8 @@ linux-kernel-headers: robostack: [] log4cxx: robostack: [log4cxx] +lua-dev: + robostack: [lua] lua5.2-dev: robostack: [lua] lz4: @@ -929,6 +935,8 @@ python3-coverage: robostack: [coverage] python3-cryptography: robostack: [cryptography] +python3-dbus: + robostack: [dbus-python] python3-defusedxml: robostack: [defusedxml] python3-deprecated: @@ -1058,6 +1066,8 @@ python3-pydantic: robostack: [pydantic] python3-pydot: robostack: [pydot] +python3-pygnssutils-pip: + robostack: [pygnssutils] python3-pygraphviz: robostack: [graphviz, pygraphviz] python3-pykdl: @@ -1126,6 +1136,8 @@ python3-sklearn: robostack: [scikit-learn] python3-sphinx: robostack: [sphinx] +python3-tabulate: + robostack: [tabulate] python3-termcolor: robostack: [termcolor] python3-texttable: @@ -1150,6 +1162,8 @@ python3-twisted: robostack: [twisted] python3-typeguard: robostack: [typeguard] +python3-types-pyyaml: + robostack: [types-pyyaml] python3-typing-extensions: robostack: [typing_extensions] python3-ujson: @@ -1229,6 +1243,8 @@ rti-connext-dds-5.3.1: robostack: [] rti-connext-dds-6.0.1: robostack: [] +rtklib: + robostack: [rtklib-ex] ruby: robostack: [ruby] sbcl: diff --git a/tools/maintenance.py b/tools/maintenance.py index 11dbc9ce..eb840108 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -45,6 +45,7 @@ "update-vinca": "move the vinca pin to the newest commit of its branch (opens a PR)", "update-pixi": "upgrade the tooling's dependencies in pixi.toml / pixi.lock to their newest versions (opens a PR)", "dependency-report": "check the dependencies against the pins, in the distribution's dependency-report issue", + "sync-pixi-mappings": "copy the rosdep mappings (robostack.yaml) to pixi-build-ros (opens a PR on prefix-dev/pixi)", "update-build-timings": "record the build times of the last week's build-branch runs, for batching by time (opens a PR)", } PER_DISTRO_COMMANDS = ("update-rosdistro-snapshot", "find-stale-packages", "update-conda-forge-pinning", @@ -1119,6 +1120,60 @@ def update_pixi(changes: Path | None = None) -> Result: return Result("Upgrade pixi dependencies", "\n".join(lines), changed=True, labels=[rs.NO_BUILD_LABEL]) +PIXI_ROS_BACKEND = Path("crates/pixi_build_ros") +PIXI_MAPPING_HEADER = """\ +# Generated by robostack-bot from {source} in https://github.com/RoboStack/robostack: +# change the mappings there, not here.{extra} +""" + + +def pixi_mappings() -> dict[str, str]: + """pixi-build-ros's copies of the rosdep mappings: shared/robostack.yaml as is, and + every distribution's own (distros//robostack.yaml, over it) under its name.""" + shared = PIXI_MAPPING_HEADER.format(source="shared/robostack.yaml", extra="") + shared += (rs.ROOT / "shared" / "robostack.yaml").read_text() + distros = PIXI_MAPPING_HEADER.format( + source="distros//robostack.yaml", + extra="\n# Each distribution's mappings, over robostack.yaml's.") + for path in sorted((rs.ROOT / "distros").glob("*/robostack.yaml")): + lines = path.read_text().splitlines() + distros += f"{path.parent.name}:\n" + "".join(f" {line}\n" if line else "\n" for line in lines) + return {"robostack.yaml": shared, "robostack_distros.yaml": distros} + + +def _mapping_changes(name: str, old: str, new: str) -> list[str]: + old_map, new_map = yaml.safe_load(old) or {}, yaml.safe_load(new) or {} + if name == "robostack_distros.yaml": # {distro: {key: mapping}} -> {"distro: key": mapping} + old_map, new_map = ({f"{d}: {k}": v for d, m in x.items() for k, v in (m or {}).items()} + for x in (old_map, new_map)) + lines = [] + for what, keys in (("added", new_map.keys() - old_map.keys()), ("removed", old_map.keys() - new_map.keys()), + ("changed", {k for k in new_map.keys() & old_map.keys() if new_map[k] != old_map[k]})): + if keys: + lines.append(f"- {what}: " + ", ".join(f"`{k}`" for k in sorted(keys))) + return lines + + +def sync_pixi_mappings(pixi: Path) -> Result: + """Write pixi_mappings() into a checkout of prefix-dev/pixi, for a PR there: one set of + rosdep mappings for RoboStack's packages and `pixi build` of ROS packages.""" + rev = subprocess.run(["git", "rev-parse", "HEAD"], cwd=rs.ROOT, capture_output=True, text=True, + check=True).stdout.strip() + lines = [] + for name, text in pixi_mappings().items(): + path = pixi / PIXI_ROS_BACKEND / name + old = path.read_text() if path.exists() else "" + if old != text: + path.write_text(text) + lines += [f"`{PIXI_ROS_BACKEND / name}`:", *(_mapping_changes(name, old, text) or ["- comments only"]), ""] + if not lines: + return Result("pixi-build-ros's rosdep mappings are up to date", "Nothing to copy.") + summary = [f"The rosdep mappings of [RoboStack/robostack@{rev[:8]}](https://github.com/RoboStack/robostack/tree/{rev}), " + "which RoboStack's packages are built with, for pixi-build-ros.", "", *lines, + "Generated by robostack-bot (`sync-pixi-mappings`): mappings are changed in RoboStack/robostack."] + return Result("pixi-build-ros: update RoboStack's rosdep mappings", "\n".join(summary), changed=True) + + REPORT_LABEL = "dependency-report" REPORT_MARKER = "" MAX_ISSUE_BODY = 60000 # GitHub's limit is 65536 characters @@ -1446,6 +1501,8 @@ def main(command: str, argv: list[str], distro: str | None = None) -> int: if command == "update-pixi": parser.add_argument("--changes", type=Path, help="output of an earlier `pixi upgrade --json`") parser.add_argument("--excludes", action="store_true", help="print the --exclude arguments for pixi upgrade") + if command == "sync-pixi-mappings": + parser.add_argument("pixi", type=Path, help="a checkout of prefix-dev/pixi") if command == "update-vinca": parser.add_argument("ref", nargs="?", help="vinca branch, tag or commit (default: the pinned branch)") args = parser.parse_args(argv) @@ -1465,6 +1522,8 @@ def main(command: str, argv: list[str], distro: str | None = None) -> int: return report(dependency_report(distro), args.summary) if command == "update-build-timings": return report(update_build_timings(), args.summary) + if command == "sync-pixi-mappings": + return report(sync_pixi_mappings(args.pixi), args.summary) if command == "update-pixi": if args.excludes: print(" ".join(pixi_upgrade_args()[3:])) diff --git a/tools/robostack.py b/tools/robostack.py index f2e69e00..40a93296 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -705,7 +705,8 @@ def changed_distros(base: str) -> list[str]: REPO_COMMANDS = ("check", "update-pinning", "new-distro", "parse-command", "add-package", "rosdistro-syncs", - "rebuild", "rebuild-dependents", "update-vinca", "update-build-timings", "update-pixi") + "rebuild", "rebuild-dependents", "update-vinca", "update-build-timings", "update-pixi", + "sync-pixi-mappings") def main() -> int: diff --git a/tools/tests/test_bot_commands.py b/tools/tests/test_bot_commands.py index 28f0ff21..60fb0da2 100644 --- a/tools/tests/test_bot_commands.py +++ b/tools/tests/test_bot_commands.py @@ -5,6 +5,7 @@ import json import pytest +import yaml import maintenance import robostack as rs @@ -377,3 +378,20 @@ def run(cmd, **kwargs): (tmp_path / "upgrade.json").write_text("{}") assert not maintenance.update_pixi(tmp_path / "upgrade.json").changed assert all(cmd[:2] != ["pixi", "upgrade"] for cmd in calls) # the bot upgraded before `pixi run` + + +def test_sync_pixi_mappings(tmp_path, monkeypatch): + assert jobs("@robostack-bot sync-pixi-mappings") == [ + {"command": "sync-pixi-mappings", "distro": "", "args": "", "preview": False}] + assert maintenance.parse_command("@robostack-bot sync-pixi-mappings", "NONE") == [] + backend = tmp_path / maintenance.PIXI_ROS_BACKEND + backend.mkdir(parents=True) + (backend / "robostack.yaml").write_text("gone:\n robostack: [x]\n") + result = maintenance.sync_pixi_mappings(tmp_path) + assert result.changed + assert "- removed: `gone`" in result.summary + shared = yaml.safe_load((backend / "robostack.yaml").read_text()) + assert shared == yaml.safe_load((rs.ROOT / "shared" / "robostack.yaml").read_text()) + distros = yaml.safe_load((backend / "robostack_distros.yaml").read_text()) + assert distros["humble"]["qtbase5-dev"]["robostack"]["osx"] == ["qt-main"] + assert not maintenance.sync_pixi_mappings(tmp_path).changed # nothing left to copy