From d378290765c294db78e185e9f1b2600d6e704ede Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Fri, 20 Nov 2026 14:55:13 +0000 Subject: [PATCH] Wait for gateway unregistration before stopping A terminating service replica was stopped regardless of whether gateway replicas had unregistered it. They sync every 15 seconds, so unregistration could land up to 16 seconds into the replica's 30-second termination deadline, cutting requests still routed to it. The terminating pipeline now waits until no running gateway replica has the job registered before stopping it, and asks the gateway replicas to sync right away. The wait is capped at a minute (`JobModel.unregister_deadline`), since an unreachable gateway replica never unregisters anything. Aborted jobs are not waited for. Part-of: https://github.com/dstackai/dstack/issues/4493 Assisted-by: Claude Opus 5.5 --- contributing/PROXY.md | 2 +- contributing/RUNS-AND-JOBS.md | 2 +- .../pipeline_tasks/jobs_terminating.py | 80 +++++- ...6bbd0f_add_jobmodel_unregister_deadline.py | 34 +++ src/dstack/_internal/server/models.py | 5 + .../pipeline_tasks/test_jobs_terminating.py | 241 +++++++++++++++++- 6 files changed, 355 insertions(+), 9 deletions(-) create mode 100644 src/dstack/_internal/server/migrations/versions/2026/11_20_1241_40cbdf6bbd0f_add_jobmodel_unregister_deadline.py diff --git a/contributing/PROXY.md b/contributing/PROXY.md index cc5c2f683..3f3c4549d 100644 --- a/contributing/PROXY.md +++ b/contributing/PROXY.md @@ -64,7 +64,7 @@ The in-server proxy opens an SSH connection when it needs to forward the first r The gateway opens SSH connections when each replica is registered. -When a replica is unregistered, the gateway first reloads Nginx so that new requests stop going to the replica, and only then closes its SSH connection. The connection is closed in the background so that requests already sent through it can finish: after a short grace period for Nginx workers that have not picked up the reload yet, once nothing is connected to the replica's socket (as listed in `/proc/net/unix`), or after 5 minutes at most. The `dstack` server stops the replica's container independently, so it can still cut requests that outlive the container. +When a replica is unregistered, the gateway first reloads Nginx so that new requests stop going to the replica, and only then closes its SSH connection. The connection is closed in the background so that requests already sent through it can finish: after a short grace period for Nginx workers that have not picked up the reload yet, once nothing is connected to the replica's socket (as listed in `/proc/net/unix`), or after 5 minutes at most. The `dstack` server stops the replica only after all running gateway replicas have unregistered it, or after a minute at most. It does not wait for the drain, so requests that outlive the replica are still cut. ### Communication with the `dstack` server diff --git a/contributing/RUNS-AND-JOBS.md b/contributing/RUNS-AND-JOBS.md index 6e076d272..014967e71 100644 --- a/contributing/RUNS-AND-JOBS.md +++ b/contributing/RUNS-AND-JOBS.md @@ -85,4 +85,4 @@ Services' run lifecycle has some modifications: Services' jobs lifecycle has some modifications: - During STEP 3, once the primary job of a replica is `RUNNING` and ready to receive traffic, `JobRunningPipeline` registers that replica on the gateway. If the gateway is not accessible, the job fails with a gateway-related termination reason. -- During STEP 5, `JobTerminatingPipeline` unregisters the replica from receiving requests before the job is fully cleaned up. +- During STEP 5, `JobTerminatingPipeline` does not stop the replica until every running gateway replica has unregistered it, so that requests stop arriving first. Gateway replicas unregister replicas that are no longer `RUNNING` when they sync, and the pipeline asks them to sync right away. The wait is capped at a minute, since a gateway replica that cannot be reached never unregisters anything. Aborted jobs are not waited for. The in-server proxy stops routing requests to a replica as soon as it is no longer `RUNNING`, so it needs no wait. diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py index e053e533b..06853f9a4 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py @@ -13,6 +13,7 @@ from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport from dstack._internal.core.consts import DSTACK_RUNNER_HTTP_PORT, DSTACK_SHIM_HTTP_PORT from dstack._internal.core.errors import BackendError +from dstack._internal.core.models.gateways import GatewayReplicaStatus from dstack._internal.core.models.instances import InstanceStatus, InstanceTerminationReason from dstack._internal.core.models.runs import ( JobProvisioningData, @@ -41,16 +42,21 @@ ) from dstack._internal.server.db import get_db, get_session_ctx from dstack._internal.server.models import ( + GatewayReplicaModel, InstanceModel, JobModel, ProjectModel, RunModel, + ServiceReplicaRegistrationModel, VolumeAttachmentModel, VolumeModel, ) from dstack._internal.server.services import events from dstack._internal.server.services import logs as logs_services from dstack._internal.server.services.backends import cache as backends_cache +from dstack._internal.server.services.gateways import ( + skip_gateway_replicas_min_processing_interval, +) from dstack._internal.server.services.instances import ( emit_instance_status_change_event, get_instance_ssh_private_keys, @@ -87,6 +93,11 @@ # final log flush that follows it. JOB_TERMINATION_DEADLINE = timedelta(seconds=30) +# How long a service replica waits to be unregistered from gateway replicas before it is +# terminated anyway. Gateway replicas normally unregister it within seconds, but one that cannot +# be reached keeps it registered for as long as it stays unreachable. +JOB_UNREGISTER_TIMEOUT = timedelta(minutes=1) + @dataclass class JobTerminatingPipelineItem(PipelineItem): @@ -294,6 +305,8 @@ async def process(self, item: JobTerminatingPipelineItem): and result.instance_update_map.get("status") == InstanceStatus.TERMINATING ): self._pipeline_hinter.hint_fetch(InstanceModel.__name__) + if result.sync_gateway_replicas: + self._pipeline_hinter.hint_fetch(GatewayReplicaModel.__name__) # TODO: Hint RunPipeline to quickly move run to TERMINATED. # Currently not implemented since it also requires making run eligible for processing. # (This pipeline cannot modify runs so it's not simple). @@ -309,6 +322,7 @@ class _JobUpdateMap(ItemUpdateMap, total=False): volumes_detached_at: UpdateMapDateTime registered: bool remove_at: UpdateMapDateTime + unregister_deadline: UpdateMapDateTime class _InstanceUpdateMap(ItemUpdateMap, total=False): @@ -334,6 +348,8 @@ class _ProcessResult: detached_volume_ids: set[uuid.UUID] = field(default_factory=set) unassign_event_message: Optional[str] = None graceful_stop_event_message: Optional[str] = None + sync_gateway_replicas: bool = False + """Whether to sync the run's gateway replicas without waiting for their next scheduled sync""" @dataclass @@ -514,6 +530,10 @@ async def _apply_process_result( ) return + if result.sync_gateway_replicas and job_model.run.gateway_id is not None: + # Cross-pipeline write is ok: worst case skip_min_processing_interval is overridden. + await skip_gateway_replicas_min_processing_interval(session, job_model.run.gateway_id) + if instance_model is not None and result.delete_instance: res = await session.execute( delete(InstanceModel) @@ -639,10 +659,11 @@ async def _process_terminating_job( ) -> _ProcessResult: """ Terminates the job: - 1. tells the runner to stop the job's command - 2. tells the shim to stop the container - 3. detaches the job from the instance - 4. and detaches volumes from the instance. + 1. waits until gateway replicas stop routing requests to it + 2. tells the runner to stop the job's command + 3. tells the shim to stop the container + 4. detaches the job from the instance + 5. and detaches volumes from the instance. """ instance_update_map = None if instance_model is None else _InstanceUpdateMap() result = _ProcessResult(instance_update_map=instance_update_map) @@ -673,7 +694,9 @@ async def _process_terminating_job( job_update_map=result.job_update_map, ) elif job_model.remove_at is None: - # The first terminating pass. `graceful_termination_attempts` decides whether the runner + if await _should_wait_for_unregistration(job_model=job_model, result=result): + return result + # Termination starts. `graceful_termination_attempts` decides whether the runner # is asked to stop the job; `remove_at` decides when the container is killed regardless. # The two are independent: a job that has already finished on its own is not asked to # stop, but its logs are still drained below. @@ -763,6 +786,53 @@ def _is_aborted(job_model: JobModel) -> bool: ) +async def _should_wait_for_unregistration(job_model: JobModel, result: _ProcessResult) -> bool: + """ + Whether to keep the job running until gateway replicas stop routing requests to it. + + Until then, stopping the job would fail the requests that gateway replicas still send to it. + The wait is capped by `unregister_deadline`, since a gateway replica that cannot be reached + keeps the job registered for as long as it stays unreachable. + """ + if not job_model.registered: + return False + gateway_replica_names = await _get_registered_gateway_replica_names(job_model) + if len(gateway_replica_names) == 0: + return False + now = get_current_datetime() + if job_model.unregister_deadline is None: + result.job_update_map["unregister_deadline"] = now + JOB_UNREGISTER_TIMEOUT + result.sync_gateway_replicas = True + return True + if now < job_model.unregister_deadline: + return True + logger.warning( + "%s: still registered on gateway replicas %s, terminating anyway", + fmt(job_model), + ", ".join(gateway_replica_names), + ) + return False + + +async def _get_registered_gateway_replica_names(job_model: JobModel) -> list[str]: + async with get_session_ctx() as session: + res = await session.execute( + select(GatewayReplicaModel.name) + .join( + ServiceReplicaRegistrationModel, + ServiceReplicaRegistrationModel.gateway_replica_id == GatewayReplicaModel.id, + ) + .where( + ServiceReplicaRegistrationModel.job_id == job_model.id, + ServiceReplicaRegistrationModel.is_registered == True, + # Only running gateway replicas are synced, the others would never unregister it + GatewayReplicaModel.status == GatewayReplicaStatus.RUNNING, + ) + .order_by(GatewayReplicaModel.name) + ) + return list(res.scalars().all()) + + def _has_logs_to_drain(job_model: JobModel) -> bool: """ Whether the runner may still be holding logs for this job. diff --git a/src/dstack/_internal/server/migrations/versions/2026/11_20_1241_40cbdf6bbd0f_add_jobmodel_unregister_deadline.py b/src/dstack/_internal/server/migrations/versions/2026/11_20_1241_40cbdf6bbd0f_add_jobmodel_unregister_deadline.py new file mode 100644 index 000000000..d3deb1978 --- /dev/null +++ b/src/dstack/_internal/server/migrations/versions/2026/11_20_1241_40cbdf6bbd0f_add_jobmodel_unregister_deadline.py @@ -0,0 +1,34 @@ +"""Add JobModel.unregister_deadline + +Revision ID: 40cbdf6bbd0f +Revises: 2d2692d5b3c2 +Create Date: 2026-11-20 12:41:35.026269+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +import dstack._internal.server.models + +# revision identifiers, used by Alembic. +revision = "40cbdf6bbd0f" +down_revision = "2d2692d5b3c2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("jobs", schema=None) as batch_op: + batch_op.add_column( + sa.Column( + "unregister_deadline", + dstack._internal.server.models.NaiveDateTime(), + nullable=True, + ) + ) + + +def downgrade() -> None: + with op.batch_alter_table("jobs", schema=None) as batch_op: + batch_op.drop_column("unregister_deadline") diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 27c5311ea..696924858 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -573,6 +573,11 @@ class JobModel(PipelineModelMixin, BaseModel): over its last logs. `None` until the job starts terminating, and only set for jobs that are given time to finish -- the rest are stopped on their first terminating pass. """ + unregister_deadline: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) + """`unregister_deadline` is when a terminating service replica stops waiting to be + unregistered from gateway replicas and is terminated anyway. `None` until the replica is + found still registered on a terminating pass. + """ volumes_detached_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) instance_assigned: Mapped[bool] = mapped_column(Boolean, default=False) """`instance_assigned` shows whether instance assignment has already been attempted. diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py index 081f33385..6b36841c2 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_terminating.py @@ -1,7 +1,7 @@ import asyncio import uuid from datetime import datetime, timedelta, timezone -from typing import Optional +from typing import Optional, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -11,6 +11,7 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import TaskConfiguration +from dstack._internal.core.models.gateways import GatewayReplicaStatus, GatewayStatus from dstack._internal.core.models.instances import InstanceStatus from dstack._internal.core.models.runs import ( JobStatus, @@ -27,7 +28,13 @@ JobTerminatingWorker, _get_related_instance_lock_owner, ) -from dstack._internal.server.models import InstanceModel, JobModel, VolumeAttachmentModel +from dstack._internal.server.models import ( + GatewayReplicaModel, + InstanceModel, + JobModel, + ServiceReplicaRegistrationModel, + VolumeAttachmentModel, +) from dstack._internal.server.schemas.runner import LogEvent, PullResponse from dstack._internal.server.services.runner.client import ( PeerConnectionError, @@ -36,6 +43,9 @@ ) from dstack._internal.server.testing.common import ( ComputeMockSpec, + create_backend, + create_gateway, + create_gateway_replica, create_instance, create_job, create_project, @@ -108,6 +118,68 @@ def _lock_job(job_model: JobModel): job_model.lock_owner = JobTerminatingPipeline.__name__ +async def _create_terminating_service_replica( + session: AsyncSession, + termination_reason: JobTerminationReason, + run_termination_reason: Optional[RunTerminationReason] = None, +) -> tuple[JobModel, list[GatewayReplicaModel]]: + """Creates a terminating replica of a service behind a gateway with two replicas.""" + project = await create_project(session=session) + user = await create_user(session=session) + backend = await create_backend(session=session, project_id=project.id) + gateway = await create_gateway( + session=session, + project_id=project.id, + backend_id=backend.id, + status=GatewayStatus.RUNNING, + ) + gateway_replicas = [ + await create_gateway_replica( + session=session, backend=backend, gateway_id=gateway.id, replica_num=replica_num + ) + for replica_num in range(2) + ] + instance = await create_instance(session=session, project=project, status=InstanceStatus.BUSY) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + gateway=gateway, + status=RunStatus.RUNNING if run_termination_reason is None else RunStatus.TERMINATING, + termination_reason=run_termination_reason, + ) + job = await create_job( + session=session, + run=run, + status=JobStatus.TERMINATING, + termination_reason=termination_reason, + job_provisioning_data=get_job_provisioning_data(dockerized=True), + instance=instance, + running_at=get_current_datetime() - timedelta(minutes=1), + ready=True, + registered=True, + ) + return job, gateway_replicas + + +def _register_replica( + session: AsyncSession, + job_model: JobModel, + gateway_replica_model: GatewayReplicaModel, + is_registered: bool = True, +): + session.add( + ServiceReplicaRegistrationModel( + job_id=job_model.id, + gateway_replica_id=gateway_replica_model.id, + is_registered=is_registered, + register_attempt=0 if is_registered else 1, + ) + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @pytest.mark.usefixtures("image_config_mock") @@ -664,6 +736,171 @@ async def test_does_not_wait_for_logs_of_aborted_job( assert job.status == expected_status assert job.runner_timestamp == 2 + @pytest.mark.parametrize( + ("termination_reason", "graceful_termination_attempts"), + [ + pytest.param(JobTerminationReason.SCALED_DOWN, None, id="scaled_down"), + pytest.param(JobTerminationReason.TERMINATED_BY_USER, 0, id="stopped"), + ], + ) + async def test_waits_for_replica_to_be_unregistered_from_gateway( + self, + test_db, + session: AsyncSession, + worker: JobTerminatingWorker, + termination_reason: JobTerminationReason, + graceful_termination_attempts: Optional[int], + ): + job, gateway_replicas = await _create_terminating_service_replica( + session=session, termination_reason=termination_reason + ) + job.graceful_termination_attempts = graceful_termination_attempts + # Unregistered from one gateway replica, but not from the other yet + _register_replica(session, job, gateway_replicas[1]) + _lock_job(job) + await session.commit() + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.stop_runner", + new=AsyncMock(), + ) as stop_runner, + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating._stop_container", + new=AsyncMock(return_value=True), + ) as stop_container, + ): + await worker.process(_job_to_pipeline_item(job)) + + stop_runner.assert_not_awaited() + stop_container.assert_not_awaited() + + await session.refresh(job) + assert job.status == JobStatus.TERMINATING + assert job.graceful_termination_attempts == graceful_termination_attempts + assert job.remove_at is None + assert job.unregister_deadline is not None + assert job.unregister_deadline > get_current_datetime() + timedelta(seconds=30) + # Gateway replicas are synced right away instead of on their schedule + for gateway_replica in gateway_replicas: + await session.refresh(gateway_replica) + assert gateway_replica.skip_min_processing_interval + hint_fetch = cast(Mock, worker._pipeline_hinter.hint_fetch) + hint_fetch.assert_called_once_with(GatewayReplicaModel.__name__) + + async def test_keeps_waiting_for_replica_to_be_unregistered_until_deadline( + self, test_db, session: AsyncSession, worker: JobTerminatingWorker + ): + job, gateway_replicas = await _create_terminating_service_replica( + session=session, termination_reason=JobTerminationReason.TERMINATED_BY_USER + ) + job.graceful_termination_attempts = 0 + unregister_deadline = get_current_datetime() + timedelta(seconds=10) + job.unregister_deadline = unregister_deadline + _register_replica(session, job, gateway_replicas[0]) + _lock_job(job) + await session.commit() + + with patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.stop_runner", + new=AsyncMock(), + ) as stop_runner: + await worker.process(_job_to_pipeline_item(job)) + + stop_runner.assert_not_awaited() + + await session.refresh(job) + assert job.remove_at is None + assert job.unregister_deadline == unregister_deadline + # Gateway replicas were already asked to sync when the wait started + await session.refresh(gateway_replicas[0]) + assert not gateway_replicas[0].skip_min_processing_interval + cast(Mock, worker._pipeline_hinter.hint_fetch).assert_not_called() + + @pytest.mark.parametrize( + ("registered", "gateway_replica_status", "deadline_passed"), + [ + pytest.param(False, GatewayReplicaStatus.RUNNING, False, id="unregistered"), + # A gateway replica that is not running is not synced, so it never unregisters + pytest.param(True, GatewayReplicaStatus.TERMINATING, False, id="gateway_terminating"), + # A gateway replica that cannot be reached must not hold the job forever + pytest.param(True, GatewayReplicaStatus.RUNNING, True, id="deadline_passed"), + ], + ) + async def test_stops_replica_once_unregistered_from_gateway( + self, + test_db, + session: AsyncSession, + worker: JobTerminatingWorker, + registered: bool, + gateway_replica_status: GatewayReplicaStatus, + deadline_passed: bool, + ): + job, gateway_replicas = await _create_terminating_service_replica( + session=session, termination_reason=JobTerminationReason.TERMINATED_BY_USER + ) + job.graceful_termination_attempts = 0 + job.unregister_deadline = get_current_datetime() + timedelta( + seconds=-1 if deadline_passed else 10 + ) + gateway_replicas[0].status = gateway_replica_status + if registered: + _register_replica(session, job, gateway_replicas[0]) + else: + # A failed registration leaves a record behind + _register_replica(session, job, gateway_replicas[0], is_registered=False) + _lock_job(job) + await session.commit() + + with patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.stop_runner", + new=AsyncMock(), + ) as stop_runner: + await worker.process(_job_to_pipeline_item(job)) + + stop_runner.assert_awaited_once() + + await session.refresh(job) + assert job.graceful_termination_attempts == 1 + assert job.remove_at is not None + + async def test_does_not_wait_for_aborted_replica_to_be_unregistered_from_gateway( + self, + test_db, + session: AsyncSession, + worker: JobTerminatingWorker, + ssh_tunnel_mock: Mock, + runner_client_mock: Mock, + ): + job, gateway_replicas = await _create_terminating_service_replica( + session=session, + termination_reason=JobTerminationReason.ABORTED_BY_USER, + run_termination_reason=RunTerminationReason.ABORTED_BY_USER, + ) + _register_replica(session, job, gateway_replicas[0]) + _lock_job(job) + await session.commit() + + runner_client_mock.pull.return_value = PullResponse( + job_states=[], job_logs=[], runner_logs=[], last_updated=0, has_more=True + ) + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating.logs_services.write_logs" + ), + patch( + "dstack._internal.server.background.pipeline_tasks.jobs_terminating._stop_container", + new=AsyncMock(return_value=True), + ) as stop_container, + ): + await worker.process(_job_to_pipeline_item(job)) + + stop_container.assert_awaited_once() + + await session.refresh(job) + assert job.status == JobStatus.ABORTED + assert job.unregister_deadline is None + async def test_does_not_wait_for_logs_if_runner_no_longer_runs_job( self, test_db,