Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions contributing/PROXY.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ The in-server proxy opens an SSH connection when it needs to forward the first r

The gateway opens SSH connections when each replica is registered.

When a replica is unregistered, the gateway first reloads Nginx so that new requests stop going to the replica, and only then closes its SSH connection. The connection is closed in the background so that requests already sent through it can finish: after a short grace period for Nginx workers that have not picked up the reload yet, once nothing is connected to the replica's socket (as listed in `/proc/net/unix`), or after 5 minutes at most. The `dstack` server stops the replica's container independently, so it can still cut requests that outlive the container.

### Communication with the `dstack` server

The in-server proxy is part of the `dstack` server, so no network communication takes place between them.
Expand Down
31 changes: 21 additions & 10 deletions src/dstack/_internal/proxy/gateway/services/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,13 @@ async def unregister_service(

logger.debug("Unregistering service %s", service.fmt())

await nginx.unregister(service)
# Only after the reload, so that Nginx stops routing requests to the replicas
await stop_replica_connections(
ids=(r.id for r in service.replicas),
service_conn_pool=service_conn_pool,
drain=True,
)
await nginx.unregister(service)
await repo.delete_models_by_run(project_name, run_name)
await repo.delete_service(project_name, run_name)

Expand Down Expand Up @@ -344,18 +346,16 @@ async def apply_service(
nginx: Nginx,
service_conn_pool: ServiceConnectionPool,
) -> dict[models.Replica, BaseException]:
removed_replica_ids = []
if old_service is not None:
if service.domain != old_service.domain:
raise UnexpectedProxyError(
f"Did not expect service {service.fmt()}"
f" domain name to change ({old_service.domain} -> {service.domain})"
)
await stop_replica_connections(
ids=(
replica.id for replica in old_service.replicas if replica not in service.replicas
),
service_conn_pool=service_conn_pool,
)
removed_replica_ids = [
replica.id for replica in old_service.replicas if replica not in service.replicas
]
replica_conns, replica_failures = await get_or_add_replica_connections(
service, repo, service_conn_pool
)
Expand All @@ -370,6 +370,10 @@ async def apply_service(
]
service_config = await get_nginx_service_config(service, replica_configs)
await nginx.register(service_config, (await repo.get_config()).acme_settings)
# Only after the reload, so that Nginx stops routing requests to the removed replicas
await stop_replica_connections(
ids=removed_replica_ids, service_conn_pool=service_conn_pool, drain=True
)
return replica_failures


Expand Down Expand Up @@ -402,10 +406,17 @@ async def get_or_add_replica_connections(


async def stop_replica_connections(
ids: Iterable[str], service_conn_pool: ServiceConnectionPool
ids: Iterable[str], service_conn_pool: ServiceConnectionPool, drain: bool = False
) -> None:
tasks = map(service_conn_pool.remove, ids)
results = await asyncio.gather(*tasks, return_exceptions=True)
"""
With `drain`, return right away and close each connection once requests already sent
through it finish. See `ServiceConnectionPool.remove`.
"""
ids = list(ids)
results = await asyncio.gather(
*(service_conn_pool.remove(replica_id, drain=drain) for replica_id in ids),
return_exceptions=True,
)
for replica_id, exc in zip(ids, results):
if isinstance(exc, Exception):
logger.error("Error stopping connection to replica %s: %s", replica_id, exc)
Expand Down
1 change: 1 addition & 0 deletions src/dstack/_internal/proxy/gateway/testing/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,4 @@ class Mocks:
run_certbot: AnyMock
open_conn: AnyMock
close_conn: AnyMock
wait_until_idle: AnyMock
94 changes: 91 additions & 3 deletions src/dstack/_internal/proxy/lib/services/service_connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from dstack._internal.proxy.lib.errors import UnexpectedProxyError
from dstack._internal.proxy.lib.models import Project, Replica, Service
from dstack._internal.proxy.lib.repo import BaseProxyRepo
from dstack._internal.utils.common import get_or_error
from dstack._internal.utils.common import get_or_error, run_async
from dstack._internal.utils.logging import get_logger
from dstack._internal.utils.path import FileContent

Expand All @@ -27,6 +27,13 @@
# Bound SSH startup work during a shared outage, without limiting service requests.
MAX_CONCURRENT_TUNNEL_RECONNECTS = 8
MAX_TUNNEL_RECONNECT_DELAY = 30
# A drained connection stays open for at least this long, since Nginx workers may still route
# requests to a removed replica for a short while after a reload.
REPLICA_DRAIN_GRACE_PERIOD = 5
REPLICA_DRAIN_POLL_INTERVAL = 1
# Upper bound for draining, e.g. if a WebSocket connection keeps the replica busy.
# Matches the default `stop_duration`, the time a job is given to stop gracefully.
REPLICA_DRAIN_TIMEOUT = 300


class ServiceClient(httpx.AsyncClient):
Expand Down Expand Up @@ -202,6 +209,7 @@ class ServiceConnectionPool:
def __init__(self) -> None:
# TODO(#2238): remove connections to stopped replicas in-server
self.connections: Dict[str, ServiceConnection] = {}
self._draining: set[asyncio.Task] = set()
self._reconnect_semaphore = asyncio.Semaphore(MAX_CONCURRENT_TUNNEL_RECONNECTS)

async def get(self, replica_id: str) -> Optional[ServiceConnection]:
Expand Down Expand Up @@ -239,12 +247,30 @@ async def get_or_add(
raise
return connection

async def remove(self, replica_id: str) -> None:
async def remove(self, replica_id: str, drain: bool = False) -> None:
"""
Remove the replica's connection from the pool and close it.

With `drain`, return right away and close the connection in the background once nothing
is connected to its socket, so that requests already sent to the replica can finish.
The caller must stop routing requests to the socket first, e.g. by reloading Nginx.
"""
connection = self.connections.pop(replica_id, None)
if connection is not None:
if connection is None:
return
if not drain:
await connection.close()
return
task = asyncio.create_task(_close_after_drain(connection))
# The event loop only keeps weak references to tasks
self._draining.add(task)
task.add_done_callback(self._draining.discard)

async def remove_all(self) -> None:
draining = list(self._draining)
for task in draining:
task.cancel() # closes the connection without waiting for the drain
await asyncio.gather(*draining, return_exceptions=True)
replica_ids = list(self.connections)
results = await asyncio.gather(
*(self.remove(replica_id) for replica_id in replica_ids), return_exceptions=True
Expand Down Expand Up @@ -280,3 +306,65 @@ async def get_service_replica_client(
)
connection = await service_conn_pool.get_or_add(project, service, replica)
return await connection.client()


async def _close_after_drain(connection: ServiceConnection) -> None:
replica_id = connection.replica.id
try:
if await _wait_until_idle(connection.app_socket_path):
logger.debug("Connection to replica %s is drained, closing", replica_id)
else:
logger.warning(
"Connection to replica %s is still in use after %ss, closing",
replica_id,
REPLICA_DRAIN_TIMEOUT,
)
finally:
try:
await connection.close()
except Exception:
logger.exception("Error closing drained connection to replica %s", replica_id)


async def _wait_until_idle(socket_path: Path) -> bool:
"""
Wait until nothing is connected to the Unix socket, for at most `REPLICA_DRAIN_TIMEOUT`.

Returns whether the socket became idle in time.
"""
await asyncio.sleep(REPLICA_DRAIN_GRACE_PERIOD)
waited = REPLICA_DRAIN_GRACE_PERIOD
# `None` means the connections cannot be counted, so wait for the timeout
while await _count_socket_connections(socket_path) != 0:
if waited >= REPLICA_DRAIN_TIMEOUT:
return False
await asyncio.sleep(REPLICA_DRAIN_POLL_INTERVAL)
waited += REPLICA_DRAIN_POLL_INTERVAL
return True


# `SS_CONNECTED`, the state of a connected socket in `/proc/net/unix`
_SS_CONNECTED = "03"


async def _count_socket_connections(socket_path: Path) -> Optional[int]:
"""
Count the connections accepted on a listening Unix socket, or return `None` if they cannot
be counted.

Accepted connections are listed in `/proc/net/unix` with the listening socket's path
and the connected state. Linux-only, like the gateway. Not `psutil.net_connections()`:
it reports no state for Unix sockets, so it cannot tell connections from the listener.
"""
try:
table = await run_async(Path("/proc/net/unix").read_text)
except OSError:
return None
path = str(socket_path)
count = 0
# Columns: Num RefCount Protocol Flags Type St Inode [Path]
for line in table.splitlines()[1:]:
fields = line.split(maxsplit=7)
if len(fields) == 8 and fields[7] == path and fields[5] == _SS_CONNECTED:
count += 1
return count
4 changes: 4 additions & 0 deletions src/tests/_internal/proxy/gateway/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,15 @@ def system_mocks() -> Generator[Mocks, None, None]:
patch(f"{nginx}.Nginx.run_certbot") as run_certbot,
patch(f"{connection}.ServiceConnection.open") as open_conn,
patch(f"{connection}.ServiceConnection.close") as close_conn,
patch(f"{connection}._wait_until_idle") as wait_until_idle,
):
sudo.return_value = []
# Drains finish right away unless a test holds them
wait_until_idle.return_value = True
yield Mocks(
reload_nginx=reload_nginx,
run_certbot=run_certbot,
open_conn=open_conn,
close_conn=close_conn,
wait_until_idle=wait_until_idle,
)
36 changes: 34 additions & 2 deletions src/tests/_internal/proxy/gateway/routers/test_registry.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import asyncio
import re
import uuid
from datetime import datetime, timezone
Expand Down Expand Up @@ -95,6 +96,28 @@ def sample_model_options(name: str = "test-model") -> dict:
}


def hold_drains(mocks: Mocks) -> asyncio.Event:
"""Keep the connections of removed replicas draining until the returned event is set."""
drained = asyncio.Event()

async def wait_until_idle(socket_path: Path) -> bool:
await drained.wait()
return True

mocks.wait_until_idle.side_effect = wait_until_idle
return drained


async def wait_for_closed_connections(mocks: Mocks, count: int) -> None:
"""Wait for drained connections, which are closed in the background."""

async def closed() -> None:
while mocks.close_conn.await_count < count:
await asyncio.sleep(0)

await asyncio.wait_for(closed(), timeout=5)


@pytest.mark.asyncio
class TestUpdateReplicaProxyJumpKeys:
async def test_saves_key_without_opening_a_missing_connection(self, tmp_path, system_mocks):
Expand Down Expand Up @@ -656,6 +679,7 @@ async def test_unregister_not_registered_error(
assert system_mocks.reload_nginx.call_count == 0

async def test_unregister_with_replicas(self, tmp_path: Path, system_mocks: Mocks) -> None:
drained = hold_drains(system_mocks)
client = make_client(tmp_path)
# register service
resp = await client.post(
Expand All @@ -676,7 +700,10 @@ async def test_unregister_with_replicas(self, tmp_path: Path, system_mocks: Mock
assert resp.status_code == 200
assert not (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").exists()
assert system_mocks.reload_nginx.call_count == 4
assert system_mocks.close_conn.call_count == 2
# Requests already sent to the replicas can still finish
system_mocks.close_conn.assert_not_awaited()
drained.set()
await wait_for_closed_connections(system_mocks, 2)

async def test_unregister_with_model(self, tmp_path: Path, system_mocks: Mocks) -> None:
repo = GatewayProxyRepo()
Expand All @@ -698,6 +725,7 @@ async def test_unregister_with_model(self, tmp_path: Path, system_mocks: Mocks)
@pytest.mark.asyncio
class TestUnregisterReplica:
async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None:
drained = hold_drains(system_mocks)
client = make_client(tmp_path)
# register service
resp = await client.post(
Expand Down Expand Up @@ -725,7 +753,11 @@ async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None:
assert "replica xxx-xxx" in conf
assert "replica yyy-yyy" not in conf
assert system_mocks.reload_nginx.call_count == 4
assert system_mocks.close_conn.call_count == 1
# Requests already sent to the replica can still finish
system_mocks.close_conn.assert_not_awaited()
drained.set()
await wait_for_closed_connections(system_mocks, 1)
system_mocks.wait_until_idle.assert_awaited_once()

async def test_unregister_no_replica_error(self, tmp_path: Path, system_mocks: Mocks) -> None:
client = make_client(tmp_path)
Expand Down
Loading
Loading