diff --git a/README.md b/README.md index 48b565e..8b64f29 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,10 @@ pull requests it reports `skipped`. All solver endpoints accept `POST` with a JSON body and return a JSON object containing at minimum an `"answer"` field and a `"qasm"` field with the generated circuit. +Each solve runs in its own process. It is terminated, returning `{"answer": "timeout"}`, if it is +still running after 30 seconds, or as soon as the caller closes its connection (when running under +gunicorn), so an abandoned request doesn't keep a worker busy. + ### `GET /health` Returns `{"status": "ok"}`. diff --git a/deploy/quantumsolver.nginx b/deploy/quantumsolver.nginx index 71a5f52..2cc14e2 100644 --- a/deploy/quantumsolver.nginx +++ b/deploy/quantumsolver.nginx @@ -19,9 +19,9 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - # Must exceed gunicorn timeout (30 s) to avoid nginx cutting the connection + # Must exceed gunicorn timeout (35 s) to avoid nginx cutting the connection # before gunicorn can return a timeout response. - proxy_read_timeout 35s; + proxy_read_timeout 40s; proxy_connect_timeout 5s; } } diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 2592bbc..ebc3879 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -2,13 +2,13 @@ import multiprocessing -# Each request blocks for up to 25 s (internal timeout), so sync workers are correct. +# Each request blocks for up to SOLVE_TIMEOUT (30 s, routes.py), so sync workers are correct. workers = multiprocessing.cpu_count() * 2 + 1 worker_class = "sync" -# Must exceed the 25-second in-app timeout so gunicorn doesn't kill a legitimately +# Must exceed routes.SOLVE_TIMEOUT (30 s) so gunicorn doesn't kill a legitimately # running solver before it has a chance to respond. -timeout = 30 +timeout = 35 bind = [ "unix:/run/quantumsolver/gunicorn.sock", diff --git a/integration/test_api.py b/integration/test_api.py index f949e37..8288de8 100644 --- a/integration/test_api.py +++ b/integration/test_api.py @@ -8,7 +8,7 @@ import pytest import requests -# Generous: each solve spawns a process that re-imports qiskit, and the app self-limits at 25s. +# Generous: each solve spawns a process that re-imports qiskit, and the app self-limits at 30s. TIMEOUT = 40 # ([f0, f1], expected classification) diff --git a/src/quantumsolver/maxcut_quantum.py b/src/quantumsolver/maxcut_quantum.py index 306c8f4..7fd1f66 100644 --- a/src/quantumsolver/maxcut_quantum.py +++ b/src/quantumsolver/maxcut_quantum.py @@ -34,7 +34,7 @@ from scipy.optimize import minimize # Statevector simulation costs 2^n per evaluation and COBYLA needs dozens of -# them, so keep instances well inside the router's 25 second timeout. +# them, so keep instances well inside the router's 30 second timeout. MAX_NODES = 16 REPS = 2 SHOTS = 10000 diff --git a/src/quantumsolver/routes.py b/src/quantumsolver/routes.py index efbe67b..d7b54df 100644 --- a/src/quantumsolver/routes.py +++ b/src/quantumsolver/routes.py @@ -1,6 +1,9 @@ """This is the flask router for the quantum solver app""" import multiprocessing +import select +import socket +import time from flask import request @@ -14,6 +17,12 @@ ) from quantumsolver.app import app +# The longest a solve may run before it is terminated and "timeout" is returned. +# gunicorn.conf.py's worker timeout must stay above this. +SOLVE_TIMEOUT = 30 +# How often a running solve checks whether the caller has gone away. +POLL_INTERVAL = 0.2 + @app.route("/") @app.route("/index") @@ -108,13 +117,46 @@ def solver_maxcut_quantum(): return run_with_timeout(maxcut_target, data) +def client_gone(sock): + """True once the caller has closed the connection. + + A closed TCP connection polls readable and reads EOF. MSG_PEEK leaves any + real data (e.g. a keep-alive client's next request) unread, so only EOF or + a reset counts as gone. + """ + if sock is None: + return False + readable, _, _ = select.select([sock], [], [], 0) + if not readable: + return False + try: + return sock.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) == b"" + except BlockingIOError: + return False + except OSError: + # ECONNRESET and friends: the caller is gone. + return True + + def run_with_timeout(func, data): - """run a function with a timeout using multiprocessing""" + """run a function in a separate process, terminating it after + SOLVE_TIMEOUT seconds or as soon as the caller closes the connection""" + # gunicorn exposes the client socket; the flask dev server and test client + # don't, so there disconnects go unnoticed and only the timeout applies. + sock = request.environ.get("gunicorn.socket") ctx = multiprocessing.get_context("spawn") queue = ctx.Queue() process = ctx.Process(target=func, args=(queue, data)) process.start() - process.join(25) + + deadline = time.monotonic() + SOLVE_TIMEOUT + while process.is_alive() and time.monotonic() < deadline: + process.join(POLL_INTERVAL) + if process.is_alive() and client_gone(sock): + process.terminate() + process.join() + # Nobody is left to read this. + return {"answer": "client closed connection"} if process.is_alive(): process.terminate() diff --git a/tests/test_run_with_timeout.py b/tests/test_run_with_timeout.py new file mode 100644 index 0000000..e9a8770 --- /dev/null +++ b/tests/test_run_with_timeout.py @@ -0,0 +1,120 @@ +"""Tests for run_with_timeout: the solve timeout and aborting when the caller +closes its connection.""" + +import multiprocessing +import socket +import struct +import threading +import time + +import pytest + +from quantumsolver import routes +from quantumsolver.app import app as flask_app + + +# Solve targets run in a spawned child process, so they must be importable at +# module level. +def slow_target(queue, data): + """Stands in for a long solve.""" + time.sleep(data["seconds"]) + queue.put({"answer": "finished"}) + + +def fast_target(queue, data): + queue.put({"answer": data["x"] * 2}) + + +@pytest.fixture +def connection(): + """A connected pair: `server` plays gunicorn's client socket, `client` the caller.""" + server, client = socket.socketpair() + yield server, client + server.close() + client.close() + + +def run(func, data, sock): + """Call run_with_timeout in a request context, as a route would.""" + with flask_app.test_request_context(environ_overrides={"gunicorn.socket": sock}): + return routes.run_with_timeout(func, data) + + +def test_client_gone_without_socket(): + """No socket (flask dev server, test client): never gone.""" + assert not routes.client_gone(None) + + +def test_client_gone_idle_connection(connection): + server, _ = connection + assert not routes.client_gone(server) + + +def test_client_gone_unread_data_is_not_a_close(connection): + """A keep-alive caller's next request makes the socket readable but isn't + a close, and peeking leaves the data unread.""" + server, client = connection + client.sendall(b"GET /health") + time.sleep(0.05) + assert not routes.client_gone(server) + assert server.recv(64) == b"GET /health" + + +def test_client_gone_after_close(connection): + server, client = connection + client.close() + assert routes.client_gone(server) + + +def test_client_gone_after_reset(): + """A reset (RST) counts as gone too.""" + listener = socket.create_server(("127.0.0.1", 0)) + client = socket.create_connection(listener.getsockname()) + server, _ = listener.accept() + # SO_LINGER with a zero timeout makes close() send RST instead of FIN. + client.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + client.close() + time.sleep(0.05) + try: + assert routes.client_gone(server) + finally: + server.close() + listener.close() + + +def test_aborts_when_client_closes(connection): + """Closing the connection mid-solve terminates the solve promptly.""" + server, client = connection + threading.Timer(1.0, client.close).start() + + start = time.monotonic() + result = run(slow_target, {"seconds": 20}, server) + elapsed = time.monotonic() - start + + assert result == {"answer": "client closed connection"} + assert elapsed < 5 + assert not multiprocessing.active_children() + + +def test_times_out(connection, monkeypatch): + """A solve still running at SOLVE_TIMEOUT is terminated.""" + monkeypatch.setattr(routes, "SOLVE_TIMEOUT", 1) + server, _ = connection + + start = time.monotonic() + result = run(slow_target, {"seconds": 20}, server) + + assert result == {"answer": "timeout"} + assert time.monotonic() - start < 5 + assert not multiprocessing.active_children() + + +def test_returns_answer_with_open_connection(connection): + """An open connection doesn't disturb a normal solve.""" + server, _ = connection + assert run(fast_target, {"x": 21}, server) == {"answer": 42} + + +def test_returns_answer_without_socket(): + """Without a socket the solve runs as before.""" + assert run(fast_target, {"x": 4}, None) == {"answer": 8} diff --git a/tests/test_shors.py b/tests/test_shors.py index f711fe9..df6d61d 100644 --- a/tests/test_shors.py +++ b/tests/test_shors.py @@ -17,7 +17,7 @@ (89, [89]), ] -# Subset used for API tests — small N values that complete well within the 25s timeout +# Subset used for API tests — small N values that complete well within the 30s timeout API_CASES = [(15, [3, 5]), (21, [3, 7]), (81, [3, 3, 3, 3]), (29, [29])]