Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ pull requests it reports `skipped`.

All solver endpoints accept `POST` with a JSON body and return a JSON object containing at minimum an `"answer"` field and a `"qasm"` field with the generated circuit.

Each solve runs in its own process. It is terminated, returning `{"answer": "timeout"}`, if it is
still running after 30 seconds, or as soon as the caller closes its connection (when running under
gunicorn), so an abandoned request doesn't keep a worker busy.

### `GET /health`
Returns `{"status": "ok"}`.

Expand Down
4 changes: 2 additions & 2 deletions deploy/quantumsolver.nginx
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ server {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;

# Must exceed gunicorn timeout (30 s) to avoid nginx cutting the connection
# Must exceed gunicorn timeout (35 s) to avoid nginx cutting the connection
# before gunicorn can return a timeout response.
proxy_read_timeout 35s;
proxy_read_timeout 40s;
proxy_connect_timeout 5s;
}
}
6 changes: 3 additions & 3 deletions gunicorn.conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@

import multiprocessing

# Each request blocks for up to 25 s (internal timeout), so sync workers are correct.
# Each request blocks for up to SOLVE_TIMEOUT (30 s, routes.py), so sync workers are correct.
workers = multiprocessing.cpu_count() * 2 + 1
worker_class = "sync"

# Must exceed the 25-second in-app timeout so gunicorn doesn't kill a legitimately
# Must exceed routes.SOLVE_TIMEOUT (30 s) so gunicorn doesn't kill a legitimately
# running solver before it has a chance to respond.
timeout = 30
timeout = 35

bind = [
"unix:/run/quantumsolver/gunicorn.sock",
Expand Down
2 changes: 1 addition & 1 deletion integration/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import pytest
import requests

# Generous: each solve spawns a process that re-imports qiskit, and the app self-limits at 25s.
# Generous: each solve spawns a process that re-imports qiskit, and the app self-limits at 30s.
TIMEOUT = 40

# ([f0, f1], expected classification)
Expand Down
2 changes: 1 addition & 1 deletion src/quantumsolver/maxcut_quantum.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from scipy.optimize import minimize

# Statevector simulation costs 2^n per evaluation and COBYLA needs dozens of
# them, so keep instances well inside the router's 25 second timeout.
# them, so keep instances well inside the router's 30 second timeout.
MAX_NODES = 16
REPS = 2
SHOTS = 10000
Expand Down
46 changes: 44 additions & 2 deletions src/quantumsolver/routes.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
"""This is the flask router for the quantum solver app"""

import multiprocessing
import select
import socket
import time

from flask import request

Expand All @@ -14,6 +17,12 @@
)
from quantumsolver.app import app

# The longest a solve may run before it is terminated and "timeout" is returned.
# gunicorn.conf.py's worker timeout must stay above this.
SOLVE_TIMEOUT = 30
# How often a running solve checks whether the caller has gone away.
POLL_INTERVAL = 0.2


@app.route("/")
@app.route("/index")
Expand Down Expand Up @@ -108,13 +117,46 @@ def solver_maxcut_quantum():
return run_with_timeout(maxcut_target, data)


def client_gone(sock):
"""True once the caller has closed the connection.

A closed TCP connection polls readable and reads EOF. MSG_PEEK leaves any
real data (e.g. a keep-alive client's next request) unread, so only EOF or
a reset counts as gone.
"""
if sock is None:
return False
readable, _, _ = select.select([sock], [], [], 0)
if not readable:
return False
try:
return sock.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) == b""
except BlockingIOError:
return False
except OSError:
# ECONNRESET and friends: the caller is gone.
return True


def run_with_timeout(func, data):
"""run a function with a timeout using multiprocessing"""
"""run a function in a separate process, terminating it after
SOLVE_TIMEOUT seconds or as soon as the caller closes the connection"""
# gunicorn exposes the client socket; the flask dev server and test client
# don't, so there disconnects go unnoticed and only the timeout applies.
sock = request.environ.get("gunicorn.socket")
ctx = multiprocessing.get_context("spawn")
queue = ctx.Queue()
process = ctx.Process(target=func, args=(queue, data))
process.start()
process.join(25)

deadline = time.monotonic() + SOLVE_TIMEOUT
while process.is_alive() and time.monotonic() < deadline:
process.join(POLL_INTERVAL)
if process.is_alive() and client_gone(sock):
process.terminate()
process.join()
# Nobody is left to read this.
return {"answer": "client closed connection"}

if process.is_alive():
process.terminate()
Expand Down
120 changes: 120 additions & 0 deletions tests/test_run_with_timeout.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
"""Tests for run_with_timeout: the solve timeout and aborting when the caller
closes its connection."""

import multiprocessing
import socket
import struct
import threading
import time

import pytest

from quantumsolver import routes
from quantumsolver.app import app as flask_app


# Solve targets run in a spawned child process, so they must be importable at
# module level.
def slow_target(queue, data):
"""Stands in for a long solve."""
time.sleep(data["seconds"])
queue.put({"answer": "finished"})


def fast_target(queue, data):
queue.put({"answer": data["x"] * 2})


@pytest.fixture
def connection():
"""A connected pair: `server` plays gunicorn's client socket, `client` the caller."""
server, client = socket.socketpair()
yield server, client
server.close()
client.close()


def run(func, data, sock):
"""Call run_with_timeout in a request context, as a route would."""
with flask_app.test_request_context(environ_overrides={"gunicorn.socket": sock}):
return routes.run_with_timeout(func, data)


def test_client_gone_without_socket():
"""No socket (flask dev server, test client): never gone."""
assert not routes.client_gone(None)


def test_client_gone_idle_connection(connection):
server, _ = connection
assert not routes.client_gone(server)


def test_client_gone_unread_data_is_not_a_close(connection):
"""A keep-alive caller's next request makes the socket readable but isn't
a close, and peeking leaves the data unread."""
server, client = connection
client.sendall(b"GET /health")
time.sleep(0.05)
assert not routes.client_gone(server)
assert server.recv(64) == b"GET /health"


def test_client_gone_after_close(connection):
server, client = connection
client.close()
assert routes.client_gone(server)


def test_client_gone_after_reset():
"""A reset (RST) counts as gone too."""
listener = socket.create_server(("127.0.0.1", 0))
client = socket.create_connection(listener.getsockname())
server, _ = listener.accept()
# SO_LINGER with a zero timeout makes close() send RST instead of FIN.
client.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
client.close()
time.sleep(0.05)
try:
assert routes.client_gone(server)
finally:
server.close()
listener.close()


def test_aborts_when_client_closes(connection):
"""Closing the connection mid-solve terminates the solve promptly."""
server, client = connection
threading.Timer(1.0, client.close).start()

start = time.monotonic()
result = run(slow_target, {"seconds": 20}, server)
elapsed = time.monotonic() - start

assert result == {"answer": "client closed connection"}
assert elapsed < 5
assert not multiprocessing.active_children()


def test_times_out(connection, monkeypatch):
"""A solve still running at SOLVE_TIMEOUT is terminated."""
monkeypatch.setattr(routes, "SOLVE_TIMEOUT", 1)
server, _ = connection

start = time.monotonic()
result = run(slow_target, {"seconds": 20}, server)

assert result == {"answer": "timeout"}
assert time.monotonic() - start < 5
assert not multiprocessing.active_children()


def test_returns_answer_with_open_connection(connection):
"""An open connection doesn't disturb a normal solve."""
server, _ = connection
assert run(fast_target, {"x": 21}, server) == {"answer": 42}


def test_returns_answer_without_socket():
"""Without a socket the solve runs as before."""
assert run(fast_target, {"x": 4}, None) == {"answer": 8}
2 changes: 1 addition & 1 deletion tests/test_shors.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
(89, [89]),
]

# Subset used for API tests — small N values that complete well within the 25s timeout
# Subset used for API tests — small N values that complete well within the 30s timeout
API_CASES = [(15, [3, 5]), (21, [3, 7]), (81, [3, 3, 3, 3]), (29, [29])]


Expand Down
Loading