From 9fc2dbfe309e2472e98397e2bc1183e685f3cf78 Mon Sep 17 00:00:00 2001 From: Jason Wright Date: Tue, 17 Nov 2026 13:30:07 -0700 Subject: [PATCH] feat: abort a solve when the caller closes its connection run_with_timeout blocked in process.join(25), so a solve whose caller had gone away (e.g. a Redux worker process that was killed) kept its gunicorn sync worker and a CPU busy until it finished or timed out. Wait in short steps instead and, between them, check the client socket gunicorn exposes as environ["gunicorn.socket"]: a closed connection polls readable and peeks EOF (or raises a reset), and the solve's child process is then terminated. Under the flask dev server and test client there is no socket, so only the timeout applies, as before. Raise the backstop timeout from 25 to 30 seconds (SOLVE_TIMEOUT), and with it the gunicorn worker timeout to 35 s and nginx's proxy_read_timeout to 40 s so each still exceeds the one inside it. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 + deploy/quantumsolver.nginx | 4 +- gunicorn.conf.py | 6 +- integration/test_api.py | 2 +- src/quantumsolver/maxcut_quantum.py | 2 +- src/quantumsolver/routes.py | 46 ++++++++++- tests/test_run_with_timeout.py | 120 ++++++++++++++++++++++++++++ tests/test_shors.py | 2 +- 8 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 tests/test_run_with_timeout.py diff --git a/README.md b/README.md index 48b565e..8b64f29 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,10 @@ pull requests it reports `skipped`. All solver endpoints accept `POST` with a JSON body and return a JSON object containing at minimum an `"answer"` field and a `"qasm"` field with the generated circuit. +Each solve runs in its own process. It is terminated, returning `{"answer": "timeout"}`, if it is +still running after 30 seconds, or as soon as the caller closes its connection (when running under +gunicorn), so an abandoned request doesn't keep a worker busy. + ### `GET /health` Returns `{"status": "ok"}`. diff --git a/deploy/quantumsolver.nginx b/deploy/quantumsolver.nginx index 71a5f52..2cc14e2 100644 --- a/deploy/quantumsolver.nginx +++ b/deploy/quantumsolver.nginx @@ -19,9 +19,9 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - # Must exceed gunicorn timeout (30 s) to avoid nginx cutting the connection + # Must exceed gunicorn timeout (35 s) to avoid nginx cutting the connection # before gunicorn can return a timeout response. - proxy_read_timeout 35s; + proxy_read_timeout 40s; proxy_connect_timeout 5s; } } diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 2592bbc..ebc3879 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -2,13 +2,13 @@ import multiprocessing -# Each request blocks for up to 25 s (internal timeout), so sync workers are correct. +# Each request blocks for up to SOLVE_TIMEOUT (30 s, routes.py), so sync workers are correct. workers = multiprocessing.cpu_count() * 2 + 1 worker_class = "sync" -# Must exceed the 25-second in-app timeout so gunicorn doesn't kill a legitimately +# Must exceed routes.SOLVE_TIMEOUT (30 s) so gunicorn doesn't kill a legitimately # running solver before it has a chance to respond. -timeout = 30 +timeout = 35 bind = [ "unix:/run/quantumsolver/gunicorn.sock", diff --git a/integration/test_api.py b/integration/test_api.py index f949e37..8288de8 100644 --- a/integration/test_api.py +++ b/integration/test_api.py @@ -8,7 +8,7 @@ import pytest import requests -# Generous: each solve spawns a process that re-imports qiskit, and the app self-limits at 25s. +# Generous: each solve spawns a process that re-imports qiskit, and the app self-limits at 30s. TIMEOUT = 40 # ([f0, f1], expected classification) diff --git a/src/quantumsolver/maxcut_quantum.py b/src/quantumsolver/maxcut_quantum.py index 306c8f4..7fd1f66 100644 --- a/src/quantumsolver/maxcut_quantum.py +++ b/src/quantumsolver/maxcut_quantum.py @@ -34,7 +34,7 @@ from scipy.optimize import minimize # Statevector simulation costs 2^n per evaluation and COBYLA needs dozens of -# them, so keep instances well inside the router's 25 second timeout. +# them, so keep instances well inside the router's 30 second timeout. MAX_NODES = 16 REPS = 2 SHOTS = 10000 diff --git a/src/quantumsolver/routes.py b/src/quantumsolver/routes.py index efbe67b..d7b54df 100644 --- a/src/quantumsolver/routes.py +++ b/src/quantumsolver/routes.py @@ -1,6 +1,9 @@ """This is the flask router for the quantum solver app""" import multiprocessing +import select +import socket +import time from flask import request @@ -14,6 +17,12 @@ ) from quantumsolver.app import app +# The longest a solve may run before it is terminated and "timeout" is returned. +# gunicorn.conf.py's worker timeout must stay above this. +SOLVE_TIMEOUT = 30 +# How often a running solve checks whether the caller has gone away. +POLL_INTERVAL = 0.2 + @app.route("/") @app.route("/index") @@ -108,13 +117,46 @@ def solver_maxcut_quantum(): return run_with_timeout(maxcut_target, data) +def client_gone(sock): + """True once the caller has closed the connection. + + A closed TCP connection polls readable and reads EOF. MSG_PEEK leaves any + real data (e.g. a keep-alive client's next request) unread, so only EOF or + a reset counts as gone. + """ + if sock is None: + return False + readable, _, _ = select.select([sock], [], [], 0) + if not readable: + return False + try: + return sock.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) == b"" + except BlockingIOError: + return False + except OSError: + # ECONNRESET and friends: the caller is gone. + return True + + def run_with_timeout(func, data): - """run a function with a timeout using multiprocessing""" + """run a function in a separate process, terminating it after + SOLVE_TIMEOUT seconds or as soon as the caller closes the connection""" + # gunicorn exposes the client socket; the flask dev server and test client + # don't, so there disconnects go unnoticed and only the timeout applies. + sock = request.environ.get("gunicorn.socket") ctx = multiprocessing.get_context("spawn") queue = ctx.Queue() process = ctx.Process(target=func, args=(queue, data)) process.start() - process.join(25) + + deadline = time.monotonic() + SOLVE_TIMEOUT + while process.is_alive() and time.monotonic() < deadline: + process.join(POLL_INTERVAL) + if process.is_alive() and client_gone(sock): + process.terminate() + process.join() + # Nobody is left to read this. + return {"answer": "client closed connection"} if process.is_alive(): process.terminate() diff --git a/tests/test_run_with_timeout.py b/tests/test_run_with_timeout.py new file mode 100644 index 0000000..e9a8770 --- /dev/null +++ b/tests/test_run_with_timeout.py @@ -0,0 +1,120 @@ +"""Tests for run_with_timeout: the solve timeout and aborting when the caller +closes its connection.""" + +import multiprocessing +import socket +import struct +import threading +import time + +import pytest + +from quantumsolver import routes +from quantumsolver.app import app as flask_app + + +# Solve targets run in a spawned child process, so they must be importable at +# module level. +def slow_target(queue, data): + """Stands in for a long solve.""" + time.sleep(data["seconds"]) + queue.put({"answer": "finished"}) + + +def fast_target(queue, data): + queue.put({"answer": data["x"] * 2}) + + +@pytest.fixture +def connection(): + """A connected pair: `server` plays gunicorn's client socket, `client` the caller.""" + server, client = socket.socketpair() + yield server, client + server.close() + client.close() + + +def run(func, data, sock): + """Call run_with_timeout in a request context, as a route would.""" + with flask_app.test_request_context(environ_overrides={"gunicorn.socket": sock}): + return routes.run_with_timeout(func, data) + + +def test_client_gone_without_socket(): + """No socket (flask dev server, test client): never gone.""" + assert not routes.client_gone(None) + + +def test_client_gone_idle_connection(connection): + server, _ = connection + assert not routes.client_gone(server) + + +def test_client_gone_unread_data_is_not_a_close(connection): + """A keep-alive caller's next request makes the socket readable but isn't + a close, and peeking leaves the data unread.""" + server, client = connection + client.sendall(b"GET /health") + time.sleep(0.05) + assert not routes.client_gone(server) + assert server.recv(64) == b"GET /health" + + +def test_client_gone_after_close(connection): + server, client = connection + client.close() + assert routes.client_gone(server) + + +def test_client_gone_after_reset(): + """A reset (RST) counts as gone too.""" + listener = socket.create_server(("127.0.0.1", 0)) + client = socket.create_connection(listener.getsockname()) + server, _ = listener.accept() + # SO_LINGER with a zero timeout makes close() send RST instead of FIN. + client.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + client.close() + time.sleep(0.05) + try: + assert routes.client_gone(server) + finally: + server.close() + listener.close() + + +def test_aborts_when_client_closes(connection): + """Closing the connection mid-solve terminates the solve promptly.""" + server, client = connection + threading.Timer(1.0, client.close).start() + + start = time.monotonic() + result = run(slow_target, {"seconds": 20}, server) + elapsed = time.monotonic() - start + + assert result == {"answer": "client closed connection"} + assert elapsed < 5 + assert not multiprocessing.active_children() + + +def test_times_out(connection, monkeypatch): + """A solve still running at SOLVE_TIMEOUT is terminated.""" + monkeypatch.setattr(routes, "SOLVE_TIMEOUT", 1) + server, _ = connection + + start = time.monotonic() + result = run(slow_target, {"seconds": 20}, server) + + assert result == {"answer": "timeout"} + assert time.monotonic() - start < 5 + assert not multiprocessing.active_children() + + +def test_returns_answer_with_open_connection(connection): + """An open connection doesn't disturb a normal solve.""" + server, _ = connection + assert run(fast_target, {"x": 21}, server) == {"answer": 42} + + +def test_returns_answer_without_socket(): + """Without a socket the solve runs as before.""" + assert run(fast_target, {"x": 4}, None) == {"answer": 8} diff --git a/tests/test_shors.py b/tests/test_shors.py index f711fe9..df6d61d 100644 --- a/tests/test_shors.py +++ b/tests/test_shors.py @@ -17,7 +17,7 @@ (89, [89]), ] -# Subset used for API tests — small N values that complete well within the 25s timeout +# Subset used for API tests — small N values that complete well within the 30s timeout API_CASES = [(15, [3, 5]), (21, [3, 7]), (81, [3, 3, 3, 3]), (29, [29])]