Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion contributing/PROXY.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ The in-server proxy opens an SSH connection when it needs to forward the first r

The gateway opens SSH connections when each replica is registered.

When a replica is unregistered, the gateway first reloads Nginx so that new requests stop going to the replica, and only then closes its SSH connection. The connection is closed in the background so that requests already sent through it can finish: after a short grace period for Nginx workers that have not picked up the reload yet, once nothing is connected to the replica's socket (as listed in `/proc/net/unix`), or after 5 minutes at most. The `dstack` server stops the replica's container independently, so it can still cut requests that outlive the container.
When a replica is unregistered, the gateway first reloads Nginx so that new requests stop going to the replica, and only then closes its SSH connection. The connection is closed in the background so that requests already sent through it can finish: after a short grace period for Nginx workers that have not picked up the reload yet, once nothing is connected to the replica's socket (as listed in `/proc/net/unix`), or after 5 minutes at most. The `dstack` server stops the replica only after all running gateway replicas have unregistered it, or after a minute at most. It does not wait for the drain, so requests that outlive the replica are still cut.

### Communication with the `dstack` server

Expand Down
2 changes: 1 addition & 1 deletion contributing/RUNS-AND-JOBS.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,4 +85,4 @@ Services' run lifecycle has some modifications:
Services' jobs lifecycle has some modifications:

- During STEP 3, once the primary job of a replica is `RUNNING` and ready to receive traffic, `JobRunningPipeline` registers that replica on the gateway. If the gateway is not accessible, the job fails with a gateway-related termination reason.
- During STEP 5, `JobTerminatingPipeline` unregisters the replica from receiving requests before the job is fully cleaned up.
- During STEP 5, `JobTerminatingPipeline` does not stop the replica until every running gateway replica has unregistered it, so that requests stop arriving first. Gateway replicas unregister replicas that are no longer `RUNNING` when they sync, and the pipeline asks them to sync right away. The wait is capped at a minute, since a gateway replica that cannot be reached never unregisters anything. Aborted jobs are not waited for. The in-server proxy stops routing requests to a replica as soon as it is no longer `RUNNING`, so it needs no wait.
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from dstack._internal.core.backends.base.compute import ComputeWithVolumeSupport
from dstack._internal.core.consts import DSTACK_RUNNER_HTTP_PORT, DSTACK_SHIM_HTTP_PORT
from dstack._internal.core.errors import BackendError
from dstack._internal.core.models.gateways import GatewayReplicaStatus
from dstack._internal.core.models.instances import InstanceStatus, InstanceTerminationReason
from dstack._internal.core.models.runs import (
JobProvisioningData,
Expand Down Expand Up @@ -41,16 +42,21 @@
)
from dstack._internal.server.db import get_db, get_session_ctx
from dstack._internal.server.models import (
GatewayReplicaModel,
InstanceModel,
JobModel,
ProjectModel,
RunModel,
ServiceReplicaRegistrationModel,
VolumeAttachmentModel,
VolumeModel,
)
from dstack._internal.server.services import events
from dstack._internal.server.services import logs as logs_services
from dstack._internal.server.services.backends import cache as backends_cache
from dstack._internal.server.services.gateways import (
skip_gateway_replicas_min_processing_interval,
)
from dstack._internal.server.services.instances import (
emit_instance_status_change_event,
get_instance_ssh_private_keys,
Expand Down Expand Up @@ -87,6 +93,11 @@
# final log flush that follows it.
JOB_TERMINATION_DEADLINE = timedelta(seconds=30)

# How long a service replica waits to be unregistered from gateway replicas before it is
# terminated anyway. Gateway replicas normally unregister it within seconds, but one that cannot
# be reached keeps it registered for as long as it stays unreachable.
JOB_UNREGISTER_TIMEOUT = timedelta(minutes=1)


@dataclass
class JobTerminatingPipelineItem(PipelineItem):
Expand Down Expand Up @@ -294,6 +305,8 @@ async def process(self, item: JobTerminatingPipelineItem):
and result.instance_update_map.get("status") == InstanceStatus.TERMINATING
):
self._pipeline_hinter.hint_fetch(InstanceModel.__name__)
if result.sync_gateway_replicas:
self._pipeline_hinter.hint_fetch(GatewayReplicaModel.__name__)
# TODO: Hint RunPipeline to quickly move run to TERMINATED.
# Currently not implemented since it also requires making run eligible for processing.
# (This pipeline cannot modify runs so it's not simple).
Expand All @@ -309,6 +322,7 @@ class _JobUpdateMap(ItemUpdateMap, total=False):
volumes_detached_at: UpdateMapDateTime
registered: bool
remove_at: UpdateMapDateTime
unregister_deadline: UpdateMapDateTime


class _InstanceUpdateMap(ItemUpdateMap, total=False):
Expand All @@ -334,6 +348,8 @@ class _ProcessResult:
detached_volume_ids: set[uuid.UUID] = field(default_factory=set)
unassign_event_message: Optional[str] = None
graceful_stop_event_message: Optional[str] = None
sync_gateway_replicas: bool = False
"""Whether to sync the run's gateway replicas without waiting for their next scheduled sync"""


@dataclass
Expand Down Expand Up @@ -514,6 +530,10 @@ async def _apply_process_result(
)
return

if result.sync_gateway_replicas and job_model.run.gateway_id is not None:
# Cross-pipeline write is ok: worst case skip_min_processing_interval is overridden.
await skip_gateway_replicas_min_processing_interval(session, job_model.run.gateway_id)

if instance_model is not None and result.delete_instance:
res = await session.execute(
delete(InstanceModel)
Expand Down Expand Up @@ -639,10 +659,11 @@ async def _process_terminating_job(
) -> _ProcessResult:
"""
Terminates the job:
1. tells the runner to stop the job's command
2. tells the shim to stop the container
3. detaches the job from the instance
4. and detaches volumes from the instance.
1. waits until gateway replicas stop routing requests to it
2. tells the runner to stop the job's command
3. tells the shim to stop the container
4. detaches the job from the instance
5. and detaches volumes from the instance.
"""
instance_update_map = None if instance_model is None else _InstanceUpdateMap()
result = _ProcessResult(instance_update_map=instance_update_map)
Expand Down Expand Up @@ -673,7 +694,9 @@ async def _process_terminating_job(
job_update_map=result.job_update_map,
)
elif job_model.remove_at is None:
# The first terminating pass. `graceful_termination_attempts` decides whether the runner
if await _should_wait_for_unregistration(job_model=job_model, result=result):
return result
# Termination starts. `graceful_termination_attempts` decides whether the runner
# is asked to stop the job; `remove_at` decides when the container is killed regardless.
# The two are independent: a job that has already finished on its own is not asked to
# stop, but its logs are still drained below.
Expand Down Expand Up @@ -763,6 +786,53 @@ def _is_aborted(job_model: JobModel) -> bool:
)


async def _should_wait_for_unregistration(job_model: JobModel, result: _ProcessResult) -> bool:
"""
Whether to keep the job running until gateway replicas stop routing requests to it.

Until then, stopping the job would fail the requests that gateway replicas still send to it.
The wait is capped by `unregister_deadline`, since a gateway replica that cannot be reached
keeps the job registered for as long as it stays unreachable.
"""
if not job_model.registered:
return False
gateway_replica_names = await _get_registered_gateway_replica_names(job_model)
if len(gateway_replica_names) == 0:
return False
now = get_current_datetime()
if job_model.unregister_deadline is None:
result.job_update_map["unregister_deadline"] = now + JOB_UNREGISTER_TIMEOUT
result.sync_gateway_replicas = True
return True
if now < job_model.unregister_deadline:
return True
logger.warning(
"%s: still registered on gateway replicas %s, terminating anyway",
fmt(job_model),
", ".join(gateway_replica_names),
)
return False


async def _get_registered_gateway_replica_names(job_model: JobModel) -> list[str]:
async with get_session_ctx() as session:
res = await session.execute(
select(GatewayReplicaModel.name)
.join(
ServiceReplicaRegistrationModel,
ServiceReplicaRegistrationModel.gateway_replica_id == GatewayReplicaModel.id,
)
.where(
ServiceReplicaRegistrationModel.job_id == job_model.id,
ServiceReplicaRegistrationModel.is_registered == True,
# Only running gateway replicas are synced, the others would never unregister it
GatewayReplicaModel.status == GatewayReplicaStatus.RUNNING,
)
.order_by(GatewayReplicaModel.name)
)
return list(res.scalars().all())


def _has_logs_to_drain(job_model: JobModel) -> bool:
"""
Whether the runner may still be holding logs for this job.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
"""Add JobModel.unregister_deadline

Revision ID: 40cbdf6bbd0f
Revises: 2d2692d5b3c2
Create Date: 2026-11-20 12:41:35.026269+00:00

"""

import sqlalchemy as sa
from alembic import op

import dstack._internal.server.models

# revision identifiers, used by Alembic.
revision = "40cbdf6bbd0f"
down_revision = "2d2692d5b3c2"
branch_labels = None
depends_on = None


def upgrade() -> None:
with op.batch_alter_table("jobs", schema=None) as batch_op:
batch_op.add_column(
sa.Column(
"unregister_deadline",
dstack._internal.server.models.NaiveDateTime(),
nullable=True,
)
)


def downgrade() -> None:
with op.batch_alter_table("jobs", schema=None) as batch_op:
batch_op.drop_column("unregister_deadline")
5 changes: 5 additions & 0 deletions src/dstack/_internal/server/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,11 @@ class JobModel(PipelineModelMixin, BaseModel):
over its last logs. `None` until the job starts terminating, and only set for jobs that are
given time to finish -- the rest are stopped on their first terminating pass.
"""
unregister_deadline: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
"""`unregister_deadline` is when a terminating service replica stops waiting to be
unregistered from gateway replicas and is terminated anyway. `None` until the replica is
found still registered on a terminating pass.
"""
volumes_detached_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)
instance_assigned: Mapped[bool] = mapped_column(Boolean, default=False)
"""`instance_assigned` shows whether instance assignment has already been attempted.
Expand Down
Loading
Loading