ci: bisect walltime shard 2 hang (do not merge)

.github/workflows/ci.yml

@@ -47,6 +47,22 @@ jobs:
           - version: "3.15t"
         pytest-version:
           - ">=8.1.1"
+        # Older pytest versions only on the latest Python
+        include:
+          # Before pytest.register_fixture (private _arg2fixturedefs override)
+          - config: headless
+            python: { version: "3.15" }
+            pytest-version: "~=9.0.0"
+          - config: pytest-benchmark-5
+            python: { version: "3.15" }
+            pytest-version: "~=9.0.0"
+          # Before 8.1.1 (getfixturedefs takes a nodeid)
+          - config: headless
+            python: { version: "3.15" }
+            pytest-version: "~=8.0.0"
+          - config: pytest-benchmark-4
+            python: { version: "3.15" }
+            pytest-version: "~=8.0.0"
 
     steps:
       - uses: actions/checkout@v5
@@ -57,7 +73,7 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "${{ matrix.python.version }}"
-      - if: matrix.config == 'valgrind' || matrix.config == 'pytest-benchmark'
+      - if: matrix.config == 'valgrind' || startsWith(matrix.config, 'pytest-benchmark')
         name: Install valgrind
         run: |
           sudo apt-get update

.github/workflows/codspeed.yml

@@ -17,16 +17,31 @@ permissions:
 jobs:
   benchmarks:
     strategy:
+      fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4]
-        mode: ["simulation", "walltime", "memory"]
+        shard: [2]
+        mode: ["walltime"]
+        variant: [pr-code-no-mp, pr-code-only-mp]
+        include:
+          - variant: pr-code-no-mp
+            kexpr: "not test_multiprocessing_map"
+          - variant: pr-code-only-mp
+            kexpr: "test_multiprocessing_map"
 
-    name: "Run ${{ matrix.mode }} benchmarks (Shard #${{ matrix.shard }})"
+    name: "Shard #${{ matrix.shard }} ${{ matrix.variant }}"
     runs-on: ${{ matrix.mode == 'walltime' && 'codspeed-macro' || 'ubuntu-24.04' }}
+    timeout-minutes: 12
     steps:
       - uses: actions/checkout@v5
         with:
           submodules: "recursive"
+      - name: Apply variant
+        run: |
+          git fetch -q --depth=1 origin master
+          if [ "${{ matrix.variant }}" = master-src ]; then
+            git checkout FETCH_HEAD -- src/pytest_codspeed/plugin.py src/pytest_codspeed/utils.py
+          fi
+          git status --short
       - uses: astral-sh/setup-uv@v7
       - uses: actions/setup-python@v6
         with:
@@ -37,11 +52,40 @@ jobs:
           sudo apt-get install valgrind -y
           uv sync --dev
           sudo apt-get remove valgrind -y
+      - name: Arm drain probe
+        run: |
+          sudo apt-get install -y strace >/dev/null
+          cat > /tmp/probe.sh <<'EOF'
+          until pgrep -f "pytest tests/benchmarks" >/dev/null; do sleep 0.5; done
+          while pgrep -f "pytest tests/benchmarks" >/dev/null; do sleep 0.2; done
+          p=$(pgrep -f "codspeed samply record" | head -n1)
+          {
+            echo "pytest exited at $(date -u +%T.%N), samply pid $p"
+            [ -n "$p" ] || exit 0
+            echo "fds: $(sudo ls /proc/$p/fd | wc -l), perf fds: $(sudo ls -l /proc/$p/fd | grep -c perf_event)"
+            t=$(ps -L -o tid=,pcpu= -p "$p" | sort -k2 -nr | head -n1 | awk '{print $1}')
+            echo "busiest tid $t"
+            for i in $(seq 20); do
+              echo "-- $(date -u +%T.%N) syscall: $(sudo cat /proc/$p/task/$t/syscall 2>&1)"
+              sudo cat /proc/$p/task/$t/stack 2>&1 | head -n 8
+              sleep 0.2
+            done
+            echo "== strace -c 10s"
+            sudo timeout -s INT 10 strace -f -c -p "$p" 2>&1 | head -n 30
+            echo "== raw strace 3s on tid $t"
+            sudo timeout -s INT 3 strace -tt -p "$t" 2>&1 | head -n 200
+            echo "== still alive at $(date -u +%T.%N)? $(pgrep -f 'codspeed samply record' || echo no)"
+          } > /tmp/drain-probe.txt 2>&1
+          EOF
+          setsid nohup bash /tmp/probe.sh >/dev/null 2>&1 &
       - name: Run benchmarks
         uses: CodSpeedHQ/action@main
         with:
           mode: ${{ matrix.mode }}
-          run: uv run pytest tests/benchmarks/ --codspeed --test-group=${{ matrix.shard }} --test-group-count=${{ env.SHARDS }}
+          run: uv run pytest tests/benchmarks/ --codspeed --test-group=${{ matrix.shard }} --test-group-count=${{ env.SHARDS }} -k "${{ matrix.kexpr }}"
+      - name: Print drain probe
+        if: always()
+        run: cat /tmp/drain-probe.txt || echo "no probe output"
 
   all-checks:
     runs-on: ubuntu-latest

src/pytest_codspeed/plugin.py

@@ -23,6 +23,7 @@
 from pytest_codspeed.instruments import MeasurementMode, get_instrument_from_mode
 from pytest_codspeed.utils import (
     BEFORE_PYTEST_8_1_1,
+    HAS_PYTEST_REGISTER_FIXTURE,
     IS_PYTEST_BENCHMARK_INSTALLED,
     IS_PYTEST_SPEED_INSTALLED,
     get_environment_metadata,
@@ -181,20 +182,40 @@ def pytest_configure(config: pytest.Config):
     config.pluginmanager.register(plugin, PLUGIN_NAME)
 
 
-@pytest.hookimpl()
-def pytest_plugin_registered(plugin, manager: pytest.PytestPluginManager):
-    """
-    Patch the benchmark fixture to use the codspeed one if codspeed is enabled and an
-    alternative benchmark fixture is available
-    """
-    if (IS_PYTEST_BENCHMARK_INSTALLED or IS_PYTEST_SPEED_INSTALLED) and isinstance(
-        plugin, FixtureManager
-    ):
-        fixture_manager = plugin
-        codspeed_plugin: CodSpeedPlugin = cast(
-            "CodSpeedPlugin", manager.get_plugin(PLUGIN_NAME)
-        )
-        if codspeed_plugin.is_codspeed_enabled:
+def _should_override_benchmark_fixture(config: pytest.Config) -> bool:
+    if not (IS_PYTEST_BENCHMARK_INSTALLED or IS_PYTEST_SPEED_INSTALLED):
+        return False
+    codspeed_plugin = cast(
+        "CodSpeedPlugin | None", config.pluginmanager.get_plugin(PLUGIN_NAME)
+    )
+    return codspeed_plugin is not None and codspeed_plugin.is_codspeed_enabled
+
+
+if HAS_PYTEST_REGISTER_FIXTURE:
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_sessionstart(session: pytest.Session):
+        """
+        Override the benchmark fixture of pytest-benchmark/pytest-speed with the
+        codspeed one. Registered later at the same (session) level, so it wins.
+        """
+        if _should_override_benchmark_fixture(session.config):
+            pytest.register_fixture(
+                name="benchmark", func=_benchmark_compat, node=session
+            )
+
+else:
+
+    @pytest.hookimpl()
+    def pytest_plugin_registered(plugin, manager: pytest.PytestPluginManager):
+        """
+        Patch the benchmark fixture to use the codspeed one if codspeed is enabled and
+        an alternative benchmark fixture is available
+        """
+        if isinstance(plugin, FixtureManager) and _should_override_benchmark_fixture(
+            plugin.config
+        ):
+            fixture_manager = plugin
             codspeed_benchmark_fixtures = plugin.getfixturedefs(
                 "codspeed_benchmark",
                 fixture_manager.session.nodeid
@@ -433,11 +454,12 @@ def codspeed_benchmark(request: pytest.FixtureRequest) -> Callable:
     return BenchmarkFixture(request)
 
 
-if not IS_PYTEST_BENCHMARK_INSTALLED:
+def _benchmark_compat(codspeed_benchmark, request: pytest.FixtureRequest):
+    """
+    Compatibility with pytest-benchmark
+    """
+    return codspeed_benchmark
 
-    @pytest.fixture(scope="function")
-    def benchmark(codspeed_benchmark, request: pytest.FixtureRequest):
-        """
-        Compatibility with pytest-benchmark
-        """
-        return codspeed_benchmark
+
+if not IS_PYTEST_BENCHMARK_INSTALLED:
+    benchmark = pytest.fixture(scope="function", name="benchmark")(_benchmark_compat)

src/pytest_codspeed/utils.py

@@ -19,6 +19,8 @@
 IS_PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None
 IS_PYTEST_SPEED_INSTALLED = importlib.util.find_spec("pytest_speed") is not None
 BEFORE_PYTEST_8_1_1 = pytest.version_tuple < (8, 1, 1)
+# Public API since pytest 9.1, required since 9.2 removed `_arg2fixturedefs`
+HAS_PYTEST_REGISTER_FIXTURE = hasattr(pytest, "register_fixture")
 SUPPORTS_PERF_TRAMPOLINE = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") == 1
 
 

tests/conftest.py

@@ -23,6 +23,11 @@
 skip_with_pytest_benchmark = pytest.mark.skipif(
     IS_PYTEST_BENCHMARK_INSTALLED, reason="pytest_benchmark installed"
 )
+xfail_pytest_benchmark_on_pytest_9_2 = pytest.mark.xfail(
+    IS_PYTEST_BENCHMARK_INSTALLED and pytest.version_tuple >= (9, 2),
+    reason="pytest-benchmark is incompatible with pytest 9.2 (Node._nodeid removal)",
+    strict=False,
+)
 if IS_PYTEST_BENCHMARK_INSTALLED:
     pytest_plugins.append("pytest_benchmark")
     print(

tests/test_pytest_plugin.py

@@ -6,6 +6,7 @@
     skip_with_perf_trampoline,
     skip_without_pytest_benchmark,
     skip_without_valgrind,
+    xfail_pytest_benchmark_on_pytest_9_2,
 )
 
 
@@ -119,6 +120,7 @@ def test_child(self):
     )
 
 
+@xfail_pytest_benchmark_on_pytest_9_2
 def test_plugin_disabled(pytester: pytest.Pytester) -> None:
     pytester.copy_example("tests/examples/test_addition_fixture.py")
     result = pytester.runpytest()
@@ -191,6 +193,7 @@ def test_another_useless_thing():
     )
 
 
+@xfail_pytest_benchmark_on_pytest_9_2
 @skip_without_pytest_benchmark
 def test_pytest_benchmark_compatibility(pytester: pytest.Pytester) -> None:
     pytester.makepyfile(