ci: control for samply fix, runner built from main (do not merge)

.github/workflows/ci.yml

@@ -47,6 +47,22 @@ jobs:
           - version: "3.15t"
         pytest-version:
           - ">=8.1.1"
+        # Older pytest versions only on the latest Python
+        include:
+          # Before pytest.register_fixture (private _arg2fixturedefs override)
+          - config: headless
+            python: { version: "3.15" }
+            pytest-version: "~=9.0.0"
+          - config: pytest-benchmark-5
+            python: { version: "3.15" }
+            pytest-version: "~=9.0.0"
+          # Before 8.1.1 (getfixturedefs takes a nodeid)
+          - config: headless
+            python: { version: "3.15" }
+            pytest-version: "~=8.0.0"
+          - config: pytest-benchmark-4
+            python: { version: "3.15" }
+            pytest-version: "~=8.0.0"
 
     steps:
       - uses: actions/checkout@v5
@@ -57,7 +73,7 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "${{ matrix.python.version }}"
-      - if: matrix.config == 'valgrind' || matrix.config == 'pytest-benchmark'
+      - if: matrix.config == 'valgrind' || startsWith(matrix.config, 'pytest-benchmark')
         name: Install valgrind
         run: |
           sudo apt-get update

.github/workflows/codspeed.yml

@@ -17,12 +17,15 @@ permissions:
 jobs:
   benchmarks:
     strategy:
+      fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4]
-        mode: ["simulation", "walltime", "memory"]
+        shard: [2]
+        mode: ["walltime"]
+        attempt: [1, 2]
 
-    name: "Run ${{ matrix.mode }} benchmarks (Shard #${{ matrix.shard }})"
+    name: "Run ${{ matrix.mode }} benchmarks (Shard #${{ matrix.shard }}, attempt ${{ matrix.attempt }})"
     runs-on: ${{ matrix.mode == 'walltime' && 'codspeed-macro' || 'ubuntu-24.04' }}
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v5
         with:
@@ -37,11 +40,44 @@ jobs:
           sudo apt-get install valgrind -y
           uv sync --dev
           sudo apt-get remove valgrind -y
+      - name: Arm hang watcher
+        run: |
+          sudo apt-get install -y gdb >/dev/null
+          cat > /tmp/watch.sh <<'EOF'
+          sleep 480
+          {
+            date -u
+            ps -ef --forest
+            systemd-cgls --no-pager -u codspeed.slice || true
+            for p in $(pgrep -x codspeed); do
+              echo "===== codspeed pid $p: $(tr '\0' ' ' < /proc/$p/cmdline)"
+              ps -L -o tid,pcpu,stat,wchan:32,comm -p "$p"
+              for t in /proc/$p/task/*; do echo "$t $(awk '{print $14, $15}' $t/stat)"; done
+              sleep 10
+              for t in /proc/$p/task/*; do echo "$t $(awk '{print $14, $15}' $t/stat)"; done
+              sudo gdb -p "$p" -batch -ex "thread apply all bt 40" 2>&1 | grep -E "^(#|Thread)"
+            done
+          } > /tmp/hang-dump.txt 2>&1
+          EOF
+          setsid nohup bash /tmp/watch.sh >/dev/null 2>&1 &
+      - name: Update Rust toolchain for the runner build
+        run: |
+          source $HOME/.cargo/env
+          cargo --version
+          rustup toolchain install stable --profile minimal
+          rustup default stable
+          cargo --version
       - name: Run benchmarks
         uses: CodSpeedHQ/action@main
+        env:
+          CODSPEED_LOG: debug
         with:
+          runner-version: branch:main
           mode: ${{ matrix.mode }}
           run: uv run pytest tests/benchmarks/ --codspeed --test-group=${{ matrix.shard }} --test-group-count=${{ env.SHARDS }}
+      - name: Print hang dump
+        if: always()
+        run: cat /tmp/hang-dump.txt || echo "no dump (no hang at 8 min)"
 
   all-checks:
     runs-on: ubuntu-latest

src/pytest_codspeed/plugin.py

@@ -23,6 +23,7 @@
 from pytest_codspeed.instruments import MeasurementMode, get_instrument_from_mode
 from pytest_codspeed.utils import (
     BEFORE_PYTEST_8_1_1,
+    HAS_PYTEST_REGISTER_FIXTURE,
     IS_PYTEST_BENCHMARK_INSTALLED,
     IS_PYTEST_SPEED_INSTALLED,
     get_environment_metadata,
@@ -181,20 +182,40 @@ def pytest_configure(config: pytest.Config):
     config.pluginmanager.register(plugin, PLUGIN_NAME)
 
 
-@pytest.hookimpl()
-def pytest_plugin_registered(plugin, manager: pytest.PytestPluginManager):
-    """
-    Patch the benchmark fixture to use the codspeed one if codspeed is enabled and an
-    alternative benchmark fixture is available
-    """
-    if (IS_PYTEST_BENCHMARK_INSTALLED or IS_PYTEST_SPEED_INSTALLED) and isinstance(
-        plugin, FixtureManager
-    ):
-        fixture_manager = plugin
-        codspeed_plugin: CodSpeedPlugin = cast(
-            "CodSpeedPlugin", manager.get_plugin(PLUGIN_NAME)
-        )
-        if codspeed_plugin.is_codspeed_enabled:
+def _should_override_benchmark_fixture(config: pytest.Config) -> bool:
+    if not (IS_PYTEST_BENCHMARK_INSTALLED or IS_PYTEST_SPEED_INSTALLED):
+        return False
+    codspeed_plugin = cast(
+        "CodSpeedPlugin | None", config.pluginmanager.get_plugin(PLUGIN_NAME)
+    )
+    return codspeed_plugin is not None and codspeed_plugin.is_codspeed_enabled
+
+
+if HAS_PYTEST_REGISTER_FIXTURE:
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_sessionstart(session: pytest.Session):
+        """
+        Override the benchmark fixture of pytest-benchmark/pytest-speed with the
+        codspeed one. Registered later at the same (session) level, so it wins.
+        """
+        if _should_override_benchmark_fixture(session.config):
+            pytest.register_fixture(
+                name="benchmark", func=_benchmark_compat, node=session
+            )
+
+else:
+
+    @pytest.hookimpl()
+    def pytest_plugin_registered(plugin, manager: pytest.PytestPluginManager):
+        """
+        Patch the benchmark fixture to use the codspeed one if codspeed is enabled and
+        an alternative benchmark fixture is available
+        """
+        if isinstance(plugin, FixtureManager) and _should_override_benchmark_fixture(
+            plugin.config
+        ):
+            fixture_manager = plugin
             codspeed_benchmark_fixtures = plugin.getfixturedefs(
                 "codspeed_benchmark",
                 fixture_manager.session.nodeid
@@ -433,11 +454,12 @@ def codspeed_benchmark(request: pytest.FixtureRequest) -> Callable:
     return BenchmarkFixture(request)
 
 
-if not IS_PYTEST_BENCHMARK_INSTALLED:
+def _benchmark_compat(codspeed_benchmark, request: pytest.FixtureRequest):
+    """
+    Compatibility with pytest-benchmark
+    """
+    return codspeed_benchmark
 
-    @pytest.fixture(scope="function")
-    def benchmark(codspeed_benchmark, request: pytest.FixtureRequest):
-        """
-        Compatibility with pytest-benchmark
-        """
-        return codspeed_benchmark
+
+if not IS_PYTEST_BENCHMARK_INSTALLED:
+    benchmark = pytest.fixture(scope="function", name="benchmark")(_benchmark_compat)

src/pytest_codspeed/utils.py

@@ -19,6 +19,8 @@
 IS_PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None
 IS_PYTEST_SPEED_INSTALLED = importlib.util.find_spec("pytest_speed") is not None
 BEFORE_PYTEST_8_1_1 = pytest.version_tuple < (8, 1, 1)
+# Public API since pytest 9.1, required since 9.2 removed `_arg2fixturedefs`
+HAS_PYTEST_REGISTER_FIXTURE = hasattr(pytest, "register_fixture")
 SUPPORTS_PERF_TRAMPOLINE = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") == 1
 
 

tests/conftest.py

@@ -23,6 +23,11 @@
 skip_with_pytest_benchmark = pytest.mark.skipif(
     IS_PYTEST_BENCHMARK_INSTALLED, reason="pytest_benchmark installed"
 )
+xfail_pytest_benchmark_on_pytest_9_2 = pytest.mark.xfail(
+    IS_PYTEST_BENCHMARK_INSTALLED and pytest.version_tuple >= (9, 2),
+    reason="pytest-benchmark is incompatible with pytest 9.2 (Node._nodeid removal)",
+    strict=False,
+)
 if IS_PYTEST_BENCHMARK_INSTALLED:
     pytest_plugins.append("pytest_benchmark")
     print(

tests/test_pytest_plugin.py

@@ -6,6 +6,7 @@
     skip_with_perf_trampoline,
     skip_without_pytest_benchmark,
     skip_without_valgrind,
+    xfail_pytest_benchmark_on_pytest_9_2,
 )
 
 
@@ -119,6 +120,7 @@ def test_child(self):
     )
 
 
+@xfail_pytest_benchmark_on_pytest_9_2
 def test_plugin_disabled(pytester: pytest.Pytester) -> None:
     pytester.copy_example("tests/examples/test_addition_fixture.py")
     result = pytester.runpytest()
@@ -191,6 +193,7 @@ def test_another_useless_thing():
     )
 
 
+@xfail_pytest_benchmark_on_pytest_9_2
 @skip_without_pytest_benchmark
 def test_pytest_benchmark_compatibility(pytester: pytest.Pytester) -> None:
     pytester.makepyfile(