diff --git a/pyhap/media/worker.py b/pyhap/media/worker.py index b65f874e..738bcdbf 100644 --- a/pyhap/media/worker.py +++ b/pyhap/media/worker.py @@ -40,6 +40,7 @@ import contextlib import itertools import logging +import math import os import socket import struct @@ -62,6 +63,10 @@ # Requests run inside HAP writes, whose hooks get at most 9 s; one second less # than the shortest sensible hook timeout leaves time to answer the write. DEFAULT_REQUEST_TIMEOUT = 5.0 +# The last share of a request's timeout is waited on its own, so it starts only +# when the caller's loop runs: after the loop was blocked, the reader gets that +# long to deliver an answer that arrived meanwhile. +REQUEST_TIMEOUT_TAIL = 0.1 STOP_TIMEOUT = 5.0 WORKER_MODULE = "pyhap.media._worker" # A worker process logs with this logger, at the parent's effective level of @@ -887,10 +892,24 @@ async def _request(self, op: str, generation: int, **fields: Any) -> Any: raise MediaWorkerError(f"Request {op} not sent: {err}") from err future: asyncio.Future[Any] = asyncio.get_running_loop().create_future() self._pending[request_id] = future + timeout = self._request_timeout + # Only a finite timeout has a last share to wait on its own. + tail = ( + timeout * REQUEST_TIMEOUT_TAIL + if timeout is not None and math.isfinite(timeout) + else 0 + ) try: - return await asyncio.wait_for( - self._exchange(runner.writer, data, future), self._request_timeout - ) + try: + async with asyncio.timeout(None if timeout is None else timeout - tail): + return await self._exchange(runner.writer, data, future) + except TimeoutError: + # After the caller's loop was blocked the answer may sit unread + # in the socket, see REQUEST_TIMEOUT_TAIL. + await asyncio.wait((future,), timeout=tail) + if not future.done(): + raise + return future.result() except (ConnectionError, BrokenPipeError) as err: raise MediaWorkerError("Media worker is not running") from err except TimeoutError as err: @@ -901,15 +920,21 @@ async def _request(self, op: str, generation: int, **fields: Any) -> Any: raise MediaWorkerError(f"Media worker did not answer {op}") from err finally: self._pending.pop(request_id, None) + future.cancel() @staticmethod async def _exchange( writer: asyncio.StreamWriter, data: bytes, future: asyncio.Future[Any] ) -> Any: - """Send a request and wait for its answer, under one deadline.""" + """Send a request and wait for its answer. + + The answer is waited for without cancelling it: a deadline that + passed must not cancel the answer the reader is about to deliver. + """ writer.write(data) await writer.drain() - return await future + await asyncio.wait((future,)) + return future.result() async def _read_loop( self, runner: _ProcessRunner | _ThreadRunner, generation: int diff --git a/tests/test_media_worker.py b/tests/test_media_worker.py index 6ef80ee3..63832635 100644 --- a/tests/test_media_worker.py +++ b/tests/test_media_worker.py @@ -5,12 +5,14 @@ import asyncio from collections.abc import AsyncIterator import contextlib +import functools import logging +import math import os import pathlib import signal import time -from typing import Any +from typing import Any, cast import pytest @@ -20,6 +22,7 @@ MAX_MESSAGE, MediaWorker, MediaWorkerError, + RemoteMediaSource, RemoteWebRTCSender, encode_message, log_file, @@ -380,6 +383,83 @@ def ignore_message(message: dict[str, Any], blob: bytes) -> None: """Drop a message from the worker, as if it never answered.""" +@pytest.mark.parametrize("separate_process", [False, True], ids=["thread", "process"]) +async def test_blocked_caller_keeps_the_worker( + separate_process: bool, tmp_path: pathlib.Path +) -> None: + """Verify a request answered while the caller's loop was blocked succeeds.""" + worker = MediaWorker( + separate_process=separate_process, log_dir=tmp_path, request_timeout=0.5 + ) + running = await worker.open_source() + source = await open_source_with_blocked_loop(worker) + assert not source.closed + assert not running.closed + assert worker.running + await worker.close() + + +async def test_blocked_caller_waits_for_a_late_answer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Verify an answer that comes in just after the caller's loop ran again counts.""" + worker = MediaWorker(request_timeout=0.5) + await worker.start() + # As from a worker thread that was held up together with the caller's loop. + late = functools.partial( + asyncio.get_running_loop().call_later, 0.01, worker._dispatch + ) + monkeypatch.setattr(worker, "_dispatch", late) + source = await open_source_with_blocked_loop(worker) + assert not source.closed + await worker.close() + + +@pytest.mark.parametrize("separate_process", [False, True], ids=["thread", "process"]) +async def test_stuck_worker_with_requests_in_their_tail_logs_no_error( + monkeypatch: pytest.MonkeyPatch, separate_process: bool, tmp_path: pathlib.Path +) -> None: + """Verify requests waiting out their timeout end without a loop error.""" + errors: list[dict[str, Any]] = [] + loop = asyncio.get_running_loop() + monkeypatch.setattr(loop, "call_exception_handler", errors.append) + worker = MediaWorker( + separate_process=separate_process, log_dir=tmp_path, request_timeout=0.5 + ) + await worker.start() + monkeypatch.setattr(worker, "_dispatch", ignore_message) + first = asyncio.ensure_future(worker.open_source()) + # The second is still in its tail when the first stops the worker. + await asyncio.sleep(0.05) + second = asyncio.ensure_future(worker.open_source()) + results = await asyncio.gather(first, second, return_exceptions=True) + assert all(isinstance(result, MediaWorkerError) for result in results) + await asyncio.sleep(0.1) + assert errors == [] + monkeypatch.undo() + await worker.close() + + +@pytest.mark.parametrize("timeout", [math.inf, None], ids=["inf", "none"]) +async def test_request_without_a_finite_timeout(timeout: float | None) -> None: + """Verify a worker whose requests may wait forever still answers them.""" + worker = MediaWorker(request_timeout=cast(float, timeout)) + source = await worker.open_source() + assert not source.closed + await worker.close() + + +async def open_source_with_blocked_loop(worker: MediaWorker) -> RemoteMediaSource: + """Open a source and block the loop past the timeout once the request is sent.""" + request = asyncio.ensure_future(worker.open_source()) + while not worker._pending: + await asyncio.sleep(0) + # Python 3.11 writes a request one turn of the loop after it is pending. + await asyncio.sleep(0) + time.sleep(1) + return await request + + async def test_oversized_request_is_refused( worker: MediaWorker, monkeypatch: pytest.MonkeyPatch ) -> None: