From 3aa94d27c6c2876b5c91ac1a1cd47d1a2dee15d0 Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Wed, 18 Nov 2026 21:34:37 +0100 Subject: [PATCH] Allow fleet importers to provision VMs Importers' runs can now provision new instances in imported backend fleets, up to `nodes.max`. The instances belong to the exporter's project and are provisioned with its backends. Currently, new instances can only be provisioned with VM-based backends. Support for container-based backends may be coming later (requires further consideration regarding SSH keys). --- contributing/ORPHAN-CLEANUP.md | 2 +- mkdocs/docs/concepts/exports.md | 2 +- .../_internal/core/backends/base/compute.py | 13 +- .../core/backends/daytona/compute.py | 3 +- .../core/backends/kubernetes/compute.py | 9 +- .../_internal/core/backends/lium/compute.py | 3 +- .../_internal/core/backends/runpod/compute.py | 6 +- .../_internal/core/backends/slurm/compute.py | 7 +- .../core/backends/template/compute.py.jinja | 1 + .../_internal/core/backends/vastai/compute.py | 3 +- .../pipeline_tasks/jobs_submitted.py | 64 +++++---- .../pipeline_tasks/orphan_cleanup.py | 6 +- ..._1200_2d2692d5b3c2_add_ix_jobs_fleet_id.py | 42 ++++++ src/dstack/_internal/server/models.py | 2 +- .../_internal/server/services/fleets.py | 20 +-- .../_internal/server/services/instances.py | 2 +- .../_internal/server/services/offers.py | 7 + .../server/services/orphan_cleanup.py | 6 +- .../_internal/server/services/runs/plan.py | 22 +-- src/dstack/_internal/server/services/ssh.py | 5 +- .../core/backends/daytona/test_compute.py | 1 + .../core/backends/lium/test_compute.py | 1 + .../core/backends/runpod/test_compute.py | 3 +- .../core/backends/vastai/test_compute.py | 1 + .../pipeline_tasks/test_jobs_submitted.py | 136 ++++++++++++++---- .../pipeline_tasks/test_orphan_cleanup.py | 32 ++++- .../server/services/runs/test_plan.py | 65 ++++++++- .../_internal/server/services/test_offers.py | 31 ++++ 28 files changed, 396 insertions(+), 99 deletions(-) create mode 100644 src/dstack/_internal/server/migrations/versions/2026/11_17_1200_2d2692d5b3c2_add_ix_jobs_fleet_id.py diff --git a/contributing/ORPHAN-CLEANUP.md b/contributing/ORPHAN-CLEANUP.md index adf7788eab..a8bd6f83d4 100644 --- a/contributing/ORPHAN-CLEANUP.md +++ b/contributing/ORPHAN-CLEANUP.md @@ -22,7 +22,7 @@ The pipeline processes each row at most once per cleanup interval (`DSTACK_SERVE Processing a row: -- STEP 1: Check whether the backend may have new orphans: it was never cleaned up, the cleanup mode changed, or the project had instance or job activity since the last clean cleanup (`clean_at`). If not, the row is only marked processed. Checking activity here rather than in the fetch query keeps fetching cheap regardless of project sizes, at the cost of a small query and row update per enrolled backend per interval. +- STEP 1: Check whether the backend may have new orphans: it was never cleaned up, the cleanup mode changed, or the project had instance activity or job activity in its fleets since the last clean cleanup (`clean_at`). If not, the row is only marked processed. Checking activity here rather than in the fetch query keeps fetching cheap regardless of project sizes, at the cost of a small query and row update per enrolled backend per interval. - STEP 2: List the backend's resource groups (`list_resource_groups()`). A resource group is one provisioning attempt as found in the provider: the owner resource (`primary_id`, the instance) and internal resources created with it. - STEP 3: Load the project's `ProviderRefModel` records and their instances (including soft-deleted ones) by `provider_ref`, in one statement. An instance is committed with its record, so it can't be missed while the record is seen. - STEP 4: Classify each group: diff --git a/mkdocs/docs/concepts/exports.md b/mkdocs/docs/concepts/exports.md index 72cc973d74..0f078a4d48 100644 --- a/mkdocs/docs/concepts/exports.md +++ b/mkdocs/docs/concepts/exports.md @@ -20,7 +20,7 @@ for running tasks, dev environments, and services. Imported resources appear wit Currently, [fleets](fleets.md) and [gateways](gateways.md) can be exported. ??? info "Limitations" - - Importer projects can only use the existing instances of imported [backend fleets](fleets.md#backend-fleets). New instances are not provisioned for importers' runs. + - [Container-based](backends.md#container-based) backends of the exporter project cannot be used for fleet importers' runs. - Runs that mount [network volumes](volumes.md#network-volumes) cannot use imported backend fleets. ??? info "Required role" diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index 1b4fbc094a..be0e089c51 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -149,6 +149,7 @@ def run_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: List[Volume], @@ -161,6 +162,11 @@ def run_job( If required to wait to get the IP address or SSH port, return partially filled `JobProvisioningData` and implement `update_provisioning_data()`. + `project_name`, `project_ssh_public_key`, and `project_ssh_private_key` belong to the + project that will own the provisioned resources -- the project of the job's fleet. It + differs from `run.project_name` if the fleet is imported from another project, so name + and tag the resources using `project_name`, not `run.project_name`. + `extra_authorized_keys` are the public keys to authorize on the job container in addition to `project_ssh_public_key`, as decided by the caller -- typically the user key, or nothing if the server does not let the user connect to the container directly. The project @@ -432,6 +438,7 @@ def run_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: List[Volume], @@ -447,7 +454,7 @@ def run_job( submits the keys to the shim later, see `Compute.run_job()`. """ instance_config = InstanceConfiguration( - project_name=run.project_name, + project_name=project_name, instance_name=get_job_instance_name(run, job), user=run.user, ssh_keys=[SSHKey(public=project_ssh_public_key.strip())], @@ -530,6 +537,7 @@ def run_jobs( run: Run, job_configurations: List[JobConfiguration], instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, placement_group: Optional[PlacementGroup], @@ -849,6 +857,7 @@ def generate_unique_instance_name( def generate_unique_instance_name_for_job( run: Run, job: Job, + project_name: str, max_length: int = _DEFAULT_MAX_RESOURCE_NAME_LEN, with_provider_ref: bool = False, ) -> str: @@ -860,7 +869,7 @@ def generate_unique_instance_name_for_job( """ return generate_unique_backend_name( resource_name=get_job_instance_name(run, job), - project_name=run.project_name, + project_name=project_name, max_length=max_length, provider_ref=get_job_provider_ref(job) if with_provider_ref else None, ) diff --git a/src/dstack/_internal/core/backends/daytona/compute.py b/src/dstack/_internal/core/backends/daytona/compute.py index c30a4a661a..0b504ab899 100644 --- a/src/dstack/_internal/core/backends/daytona/compute.py +++ b/src/dstack/_internal/core/backends/daytona/compute.py @@ -108,6 +108,7 @@ def run_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: List[Volume], @@ -120,7 +121,7 @@ def run_job( if not image_name or re.search(r"\s", image_name): raise ComputeError("Invalid Daytona container image name") instance_name = generate_unique_instance_name_for_job( - run, job, max_length=MAX_INSTANCE_NAME_LEN + run, job, project_name, max_length=MAX_INSTANCE_NAME_LEN ) commands = get_docker_commands( build_authorized_keys(project_ssh_public_key, extra_authorized_keys) diff --git a/src/dstack/_internal/core/backends/kubernetes/compute.py b/src/dstack/_internal/core/backends/kubernetes/compute.py index ac2d5386de..a0105291d2 100644 --- a/src/dstack/_internal/core/backends/kubernetes/compute.py +++ b/src/dstack/_internal/core/backends/kubernetes/compute.py @@ -234,6 +234,7 @@ def run_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: list[Volume], @@ -259,13 +260,13 @@ def run_job( # There is one jump pod per project that is used as an ssh proxy jump to connect # to all job pods of the same project. # The service is created here and configured later in update_provisioning_data() - jump_pod_name = f"dstack-{run.project_name}-ssh-jump-pod" + jump_pod_name = f"dstack-{project_name}-ssh-jump-pod" jump_pod_service_name = get_pod_service_name(jump_pod_name) _create_jump_pod_service_if_not_exists( api=api, apps_api=client.AppsV1Api(cluster.api_client), namespace=namespace, - project_name=run.project_name, + project_name=project_name, jump_pod_name=jump_pod_name, jump_pod_service_name=jump_pod_service_name, jump_pod_port=cluster.proxy_jump.port, @@ -277,13 +278,13 @@ def run_job( ) pod_name = generate_unique_instance_name_for_job( - run, job, max_length=LABEL_VALUE_MAX_LENGTH + run, job, project_name, max_length=LABEL_VALUE_MAX_LENGTH ) base_labels = build_base_labels( component="job", unique_name=pod_name, - project=run.project_name, + project=project_name, name=job.job_spec.job_name, user=run.user, provider_ref=get_job_provider_ref(job), diff --git a/src/dstack/_internal/core/backends/lium/compute.py b/src/dstack/_internal/core/backends/lium/compute.py index 93e7c13ac5..a854ffc259 100644 --- a/src/dstack/_internal/core/backends/lium/compute.py +++ b/src/dstack/_internal/core/backends/lium/compute.py @@ -108,6 +108,7 @@ def run_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: List[Volume], @@ -117,7 +118,7 @@ def run_job( ) -> JobProvisioningData: volume_commands = _get_volume_commands(run, job, volumes) instance_name = generate_unique_instance_name_for_job( - run, job, max_length=MAX_INSTANCE_NAME_LEN + run, job, project_name, max_length=MAX_INSTANCE_NAME_LEN ) authorized_keys = build_authorized_keys(project_ssh_public_key, extra_authorized_keys) # CLI keys also need forwarding access through Lium's root SSH service. diff --git a/src/dstack/_internal/core/backends/runpod/compute.py b/src/dstack/_internal/core/backends/runpod/compute.py index 324a74a9a6..fc9a9dff23 100644 --- a/src/dstack/_internal/core/backends/runpod/compute.py +++ b/src/dstack/_internal/core/backends/runpod/compute.py @@ -129,6 +129,7 @@ def run_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: List[Volume], @@ -137,7 +138,7 @@ def run_job( extra_authorized_keys: list[str], ) -> JobProvisioningData: instance_config = InstanceConfiguration( - project_name=run.project_name, + project_name=project_name, instance_name=get_job_instance_name(run, job), ssh_keys=[], user=run.user, @@ -277,6 +278,7 @@ def run_jobs( run: Run, job_configurations: List[JobConfiguration], instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, placement_group: Optional[PlacementGroup], @@ -290,7 +292,7 @@ def run_jobs( master_job_volumes = master_job_configuration.volumes all_volumes_names = set(v.name for jc in job_configurations for v in jc.volumes) instance_config = InstanceConfiguration( - project_name=run.project_name, + project_name=project_name, instance_name=get_job_instance_name(run, master_job), ssh_keys=[], user=run.user, diff --git a/src/dstack/_internal/core/backends/slurm/compute.py b/src/dstack/_internal/core/backends/slurm/compute.py index 44dfa8cf13..6ae156a7d9 100644 --- a/src/dstack/_internal/core/backends/slurm/compute.py +++ b/src/dstack/_internal/core/backends/slurm/compute.py @@ -136,6 +136,7 @@ def run_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: list[Volume], @@ -149,6 +150,7 @@ def run_job( run=run, job=job, instance_offer=instance_offer, + project_name=project_name, project_ssh_public_key=project_ssh_public_key, extra_authorized_keys=extra_authorized_keys, requirements=requirements, @@ -161,6 +163,7 @@ def run_jobs( run: Run, job_configurations: list[JobConfiguration], instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, placement_group: Optional[PlacementGroup], @@ -172,6 +175,7 @@ def run_jobs( run=run, job=master_job, instance_offer=instance_offer, + project_name=project_name, project_ssh_public_key=project_ssh_public_key, extra_authorized_keys=extra_authorized_keys, requirements=requirements, @@ -196,6 +200,7 @@ def _run_slurm_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, extra_authorized_keys: list[str], requirements: Requirements, @@ -216,7 +221,7 @@ def _run_slurm_job( assert run.run_spec.run_name is not None slurm_job_name = generate_unique_backend_name( resource_name=run.run_spec.run_name, - project_name=run.project_name, + project_name=project_name, max_length=SLURM_JOB_NAME_MAX_LENGTH, ) diff --git a/src/dstack/_internal/core/backends/template/compute.py.jinja b/src/dstack/_internal/core/backends/template/compute.py.jinja index 99b0336bb0..0f236ec0c2 100644 --- a/src/dstack/_internal/core/backends/template/compute.py.jinja +++ b/src/dstack/_internal/core/backends/template/compute.py.jinja @@ -80,6 +80,7 @@ class {{ backend_name }}Compute( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: List[Volume], diff --git a/src/dstack/_internal/core/backends/vastai/compute.py b/src/dstack/_internal/core/backends/vastai/compute.py index 96ed75e535..70a9a5d604 100644 --- a/src/dstack/_internal/core/backends/vastai/compute.py +++ b/src/dstack/_internal/core/backends/vastai/compute.py @@ -118,6 +118,7 @@ def run_job( run: Run, job: Job, instance_offer: InstanceOfferWithAvailability, + project_name: str, project_ssh_public_key: str, project_ssh_private_key: str, volumes: List[Volume], @@ -126,7 +127,7 @@ def run_job( extra_authorized_keys: list[str], ) -> JobProvisioningData: instance_name = generate_unique_instance_name_for_job( - run, job, max_length=MAX_INSTANCE_NAME_LEN + run, job, project_name, max_length=MAX_INSTANCE_NAME_LEN ) commands = get_docker_commands( build_authorized_keys(project_ssh_public_key, extra_authorized_keys) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index ed0f3ecaa2..f303525ee0 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -92,6 +92,7 @@ get_fleet_spec, get_next_instance_num, is_cloud_cluster, + new_instances_in_fleet_require_create_instance_support, ) from dstack._internal.server.services.instances import ( filter_non_placeholder_instances, @@ -580,6 +581,12 @@ async def _select_assignment( message="Could not reuse any instance for this job", ) + if preconditions.prepared_job_volumes.volumes and fleet_model.project_id != context.project.id: + return _TerminateSubmittedJobResult( + reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + message="Cannot provision instances for jobs with volumes in imported fleets", + ) + return _NewCapacityAssignment(fleet_id=fleet_model.id) @@ -663,9 +670,7 @@ async def _apply_assignment_result( await _reset_job_lock_for_retry(session=session, item=item) return fleet_spec = get_fleet_spec(fleet_model) - if not can_create_new_cloud_instance_in_fleet( - fleet_model, fleet_spec, job_project_id=context.project.id - ): + if not can_create_new_cloud_instance_in_fleet(fleet_model, fleet_spec): await _terminate_submitted_job( session=session, job_model=job_model, @@ -676,7 +681,6 @@ async def _apply_assignment_result( instance_model = _create_placeholder_instance( session=session, fleet_model=fleet_model, - project=context.project, job_model=job_model, ) session.add(instance_model) @@ -788,6 +792,11 @@ async def _load_submitted_job_context( FleetModel.instances.and_(InstanceModel.deleted == False) ) ) + .options( + joinedload(JobModel.fleet) + .joinedload(FleetModel.project) + .selectinload(ProjectModel.backends) + ) .execution_options(populate_existing=True) ) job_model = res.unique().scalar_one() @@ -884,6 +893,11 @@ async def _fetch_run_model_for_submitted_job( FleetModel.instances.and_(InstanceModel.deleted == False) ) ) + .options( + joinedload(RunModel.fleet) + .joinedload(FleetModel.project) + .selectinload(ProjectModel.backends) + ) .options(contains_eager(RunModel.jobs, alias=job_alias)) .execution_options(populate_existing=True) ) @@ -1186,7 +1200,6 @@ async def _lock_fleet_for_placeholder( def _create_placeholder_instance( session: AsyncSession, fleet_model: FleetModel, - project: ProjectModel, job_model: JobModel, ) -> InstanceModel: taken_instance_nums = {i.instance_num for i in fleet_model.instances} @@ -1197,7 +1210,7 @@ def _create_placeholder_instance( id=instance_id, name=instance_name, instance_num=instance_num, - project=project, + project_id=fleet_model.project_id, fleet=fleet_model, status=InstanceStatus.PENDING, unreachable=False, @@ -1207,7 +1220,7 @@ def _create_placeholder_instance( # if it's deleted after a failed `run_job()`. provider_ref=issue_provider_ref( session=session, - project=project, + project_id=fleet_model.project_id, owner_type=OwnerType.INSTANCE, owner_id=instance_id, owner_name=instance_name, @@ -1358,7 +1371,6 @@ async def _apply_provisioning_result( await _persist_placement_group_cleanup( session=session, fleet_model=cleanup_fleet_model, - project=cleanup_fleet_model.project, placement_group_cleanup=provisioning.placement_group_cleanup, ) await _unlock_related_fleet( @@ -1502,8 +1514,6 @@ async def _process_new_capacity_provisioning( job_model=context.job_model, run=context.run, jobs=context.jobs_to_provision, - project_ssh_public_key=context.project.ssh_public_key, - project_ssh_private_key=context.project.ssh_private_key, master_job_provisioning_data=master_provisioning_data, volumes=preconditions.prepared_job_volumes.volumes, ) @@ -1601,7 +1611,6 @@ async def _apply_new_capacity_provisioning( await _persist_placement_group_cleanup( session=session, fleet_model=fleet_model, - project=fresh_context.project, placement_group_cleanup=provisioning.placement_group_cleanup, ) @@ -1691,7 +1700,7 @@ def _resolve_provisioned_jobs_and_data( if isinstance(provisioning_data, ComputeGroupProvisioningData): compute_group_model = ComputeGroupModel( id=uuid.uuid4(), - project=context.project, + project=fleet_model.project, fleet=fleet_model, status=ComputeGroupStatus.RUNNING, provisioning_data=provisioning_data.model_dump_json(), @@ -1744,7 +1753,6 @@ async def _promote_or_create_instance_models_for_provisioned_jobs( else: instance_num = get_next_instance_num(taken_instance_nums) instance_model = _create_instance_model_for_job( - project=context.project, fleet_model=fleet_model, compute_group_model=compute_group_model, job_model=provisioned_job_model, @@ -1801,7 +1809,6 @@ def _get_job_placeholder_instance( def _create_instance_model_for_job( - project: ProjectModel, fleet_model: FleetModel, compute_group_model: Optional[ComputeGroupModel], job_model: JobModel, @@ -1821,7 +1828,7 @@ def _create_instance_model_for_job( id=uuid.uuid4(), name=f"{fleet_model.name}-{instance_num}", instance_num=instance_num, - project=project, + project=fleet_model.project, fleet=fleet_model, compute_group=compute_group_model, created_at=get_current_datetime(), @@ -2181,9 +2188,7 @@ async def _resolve_related_cluster_master_fleet( .where( FleetModel.id == fleet_id, ) - .options( - joinedload(FleetModel.project).load_only(ProjectModel.id, ProjectModel.name) - ) + .options(joinedload(FleetModel.project).joinedload(ProjectModel.backends)) .options(selectinload(FleetModel.instances.and_(InstanceModel.deleted == False))) .execution_options(populate_existing=True) .with_for_update(skip_locked=True, of=FleetModel) @@ -2391,8 +2396,6 @@ async def _provision_new_capacity( job_model: JobModel, run: Run, jobs: list[Job], - project_ssh_public_key: str, - project_ssh_private_key: str, master_job_provisioning_data: Optional[JobProvisioningData] = None, volumes: Optional[list[list[Volume]]] = None, ) -> Union[ @@ -2438,7 +2441,10 @@ async def _provision_new_capacity( fleet_model=fleet_model, ) multinode = requirements.multinode or is_multinode_job(job) - async with backends_cache.lease_project_backends(project) as backends: + # New instances belong to the fleet's project, which differs from + # the job's project if the fleet is imported. + fleet_project = fleet_model.project + async with backends_cache.lease_project_backends(fleet_project) as backends: offers = await get_offers_by_requirements( backends=backends, profile=profile, @@ -2449,6 +2455,9 @@ async def _provision_new_capacity( volumes=volumes, privileged=job.job_spec.privileged, instance_mounts=check_run_spec_requires_instance_mounts(run.run_spec), + create_instance_support=new_instances_in_fleet_require_create_instance_support( + fleet_model, job_project_id=project.id + ), placement_group=placement_group_model_to_placement_group_optional( placement_group_model ), @@ -2525,8 +2534,9 @@ async def _provision_new_capacity( run, job_configurations, offer, - project_ssh_public_key, - project_ssh_private_key, + fleet_project.name, + fleet_project.ssh_public_key, + fleet_project.ssh_private_key, placement_group_model_to_placement_group_optional(placement_group_model), requirements, extra_authorized_keys, @@ -2557,8 +2567,9 @@ async def _provision_new_capacity( run, job, offer, - project_ssh_public_key, - project_ssh_private_key, + fleet_project.name, + fleet_project.ssh_public_key, + fleet_project.ssh_private_key, offer_volumes, placement_group_model_to_placement_group_optional(placement_group_model), requirements, @@ -2708,7 +2719,6 @@ async def _load_placement_group_cleanup_fleet( async def _persist_placement_group_cleanup( session: AsyncSession, fleet_model: FleetModel, - project: ProjectModel, placement_group_cleanup: Optional[_PlacementGroupCleanup], ) -> None: if placement_group_cleanup is None: @@ -2724,7 +2734,7 @@ async def _persist_placement_group_cleanup( except_placement_group_ids=except_placement_group_ids, ) for placement_group_model in placement_group_cleanup.new_placement_group_models: - placement_group_model.project = project + placement_group_model.project = fleet_model.project placement_group_model.fleet = fleet_model placement_group_model.fleet_deleted = ( placement_group_model.id != placement_group_cleanup.selected_placement_group_id diff --git a/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py b/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py index d47d99c982..121b6bfa1c 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/orphan_cleanup.py @@ -39,6 +39,7 @@ from dstack._internal.server.db import get_db, get_session_ctx from dstack._internal.server.models import ( BackendModel, + FleetModel, InstanceModel, JobModel, OrphanCleanupModel, @@ -363,10 +364,13 @@ async def _has_activity_since( # If `run_job()` fails after creating provider resources, the job is terminated and its # placeholder instance is deleted, so only the job's activity indicates possible orphans. # Finished jobs are no longer processed, so `last_processed_at` is the finish time. + # Jobs are matched by fleet, which is set before `run_job()` and belongs to the backend's + # project even if the job belongs to another project that imports the fleet. has_job_activity = ( select(JobModel.id) + .join(FleetModel, FleetModel.id == JobModel.fleet_id) .where( - JobModel.project_id == project_id, + FleetModel.project_id == project_id, JobModel.status.in_(JobStatus.finished_statuses()), JobModel.last_processed_at >= since, ) diff --git a/src/dstack/_internal/server/migrations/versions/2026/11_17_1200_2d2692d5b3c2_add_ix_jobs_fleet_id.py b/src/dstack/_internal/server/migrations/versions/2026/11_17_1200_2d2692d5b3c2_add_ix_jobs_fleet_id.py new file mode 100644 index 0000000000..d3ba027eb9 --- /dev/null +++ b/src/dstack/_internal/server/migrations/versions/2026/11_17_1200_2d2692d5b3c2_add_ix_jobs_fleet_id.py @@ -0,0 +1,42 @@ +"""Add ix_jobs_fleet_id + +Revision ID: 2d2692d5b3c2 +Revises: 6b5610ff36c1 +Create Date: 2026-11-17 12:00:00.000000+00:00 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "2d2692d5b3c2" +down_revision = "6b5610ff36c1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.drop_index( + "ix_jobs_fleet_id", + table_name="jobs", + if_exists=True, + postgresql_concurrently=True, + ) + op.create_index( + "ix_jobs_fleet_id", + "jobs", + ["fleet_id"], + unique=False, + postgresql_concurrently=True, + ) + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + op.drop_index( + "ix_jobs_fleet_id", + table_name="jobs", + if_exists=True, + postgresql_concurrently=True, + ) diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 8801768087..27c5311ea1 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -525,7 +525,7 @@ class JobModel(PipelineModelMixin, BaseModel): run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE")) run: Mapped["RunModel"] = relationship() - fleet_id: Mapped[Optional[uuid.UUID]] = mapped_column(ForeignKey("fleets.id")) + fleet_id: Mapped[Optional[uuid.UUID]] = mapped_column(ForeignKey("fleets.id"), index=True) """`fleet_id` keeps jobs attached to fleets because we may choose an optimal fleet for a master job but not yet create an instance for it. """ diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index ea14af7114..106c43e287 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -1006,13 +1006,9 @@ def get_fleet_master_instance_provisioning_data( return None -def can_create_new_cloud_instance_in_fleet( - fleet_model: FleetModel, fleet_spec: FleetSpec, job_project_id: uuid.UUID -) -> bool: +def can_create_new_cloud_instance_in_fleet(fleet_model: FleetModel, fleet_spec: FleetSpec) -> bool: if fleet_spec.configuration.ssh_config is not None: return False - if fleet_model.project_id != job_project_id: - return False active_instances = [i for i in fleet_model.instances if i.status.is_active()] # nodes.max is a soft limit that can be exceeded when provisioning concurrently. # The fleet consolidation logic will remove redundant nodes eventually. @@ -1025,13 +1021,19 @@ def can_create_new_cloud_instance_in_fleet( return True -def check_can_create_new_cloud_instance_in_fleet( - fleet_model: FleetModel, fleet_spec: FleetSpec, job_project_id: uuid.UUID -): - if not can_create_new_cloud_instance_in_fleet(fleet_model, fleet_spec, job_project_id): +def check_can_create_new_cloud_instance_in_fleet(fleet_model: FleetModel, fleet_spec: FleetSpec): + if not can_create_new_cloud_instance_in_fleet(fleet_model, fleet_spec): raise ValueError("Cannot fit new cloud instance into fleet") +def new_instances_in_fleet_require_create_instance_support( + fleet_model: FleetModel, job_project_id: uuid.UUID +) -> bool: + # In imported fleets, only VM-based backends can provision new instances for importers' jobs. + # This limitation is to be lifted after further consideration and testing. + return fleet_model.project_id != job_project_id + + async def _create_fleet( session: AsyncSession, project: ProjectModel, diff --git a/src/dstack/_internal/server/services/instances.py b/src/dstack/_internal/server/services/instances.py index b30e2e3429..d08baf50fe 100644 --- a/src/dstack/_internal/server/services/instances.py +++ b/src/dstack/_internal/server/services/instances.py @@ -910,7 +910,7 @@ def create_instance_model( instance_id = uuid.uuid4() provider_ref = issue_provider_ref( session=session, - project=project, + project_id=project.id, owner_type=OwnerType.INSTANCE, owner_id=instance_id, owner_name=instance_name, diff --git a/src/dstack/_internal/server/services/offers.py b/src/dstack/_internal/server/services/offers.py index a0fd5e0f0f..68d193ac80 100644 --- a/src/dstack/_internal/server/services/offers.py +++ b/src/dstack/_internal/server/services/offers.py @@ -10,6 +10,7 @@ from dstack._internal.core.backends.base.backend import Backend from dstack._internal.core.backends.base.compute import ComputeWithPlacementGroupSupport from dstack._internal.core.backends.features import ( + BACKENDS_WITH_CREATE_INSTANCE_SUPPORT, BACKENDS_WITH_INSTANCE_VOLUMES_SUPPORT, BACKENDS_WITH_MULTINODE_SUPPORT, BACKENDS_WITH_PRIVILEGED_SUPPORT, @@ -44,6 +45,7 @@ async def get_offers_by_requirements( volumes: Optional[List[List[Volume]]] = None, privileged: bool = False, instance_mounts: bool = False, + create_instance_support: bool = False, placement_group: Optional[PlacementGroup] = None, blocks: Union[int, Literal["auto"]] = 1, max_offers: Optional[int] = None, @@ -88,6 +90,11 @@ async def get_offers_by_requirements( backend_types = BACKENDS_WITH_INSTANCE_VOLUMES_SUPPORT backend_types = [b for b in backend_types if b in BACKENDS_WITH_INSTANCE_VOLUMES_SUPPORT] + if create_instance_support: + if backend_types is None: + backend_types = BACKENDS_WITH_CREATE_INSTANCE_SUPPORT + backend_types = [b for b in backend_types if b in BACKENDS_WITH_CREATE_INSTANCE_SUPPORT] + if requirements.reservation is not None: if backend_types is None: backend_types = BACKENDS_WITH_RESERVATION_SUPPORT diff --git a/src/dstack/_internal/server/services/orphan_cleanup.py b/src/dstack/_internal/server/services/orphan_cleanup.py index 674662f63c..d1a1b8d0fb 100644 --- a/src/dstack/_internal/server/services/orphan_cleanup.py +++ b/src/dstack/_internal/server/services/orphan_cleanup.py @@ -4,12 +4,12 @@ from dstack._internal.core.backends.base.compute import generate_provider_ref from dstack._internal.core.models.backends.orphans import OwnerType -from dstack._internal.server.models import ProjectModel, ProviderRefModel +from dstack._internal.server.models import ProviderRefModel def issue_provider_ref( session: AsyncSession, - project: ProjectModel, + project_id: uuid.UUID, owner_type: OwnerType, owner_id: uuid.UUID, owner_name: str, @@ -23,7 +23,7 @@ def issue_provider_ref( session.add( ProviderRefModel( provider_ref=provider_ref, - project_id=project.id, + project_id=project_id, owner_type=owner_type, owner_id=owner_id, owner_name=owner_name, diff --git a/src/dstack/_internal/server/services/runs/plan.py b/src/dstack/_internal/server/services/runs/plan.py index 56590583a0..c16ecc380e 100644 --- a/src/dstack/_internal/server/services/runs/plan.py +++ b/src/dstack/_internal/server/services/runs/plan.py @@ -42,6 +42,7 @@ get_fleet_master_instance_provisioning_data, get_fleet_requirements, get_fleet_spec, + new_instances_in_fleet_require_create_instance_support, ) from dstack._internal.server.services.instances import ( filter_instances, @@ -312,6 +313,7 @@ async def select_run_candidate_fleet_models_with_filters( .where(*fleet_filters) .where(*instance_filters) .options(contains_eager(FleetModel.instances)) + .options(contains_eager(FleetModel.project).selectinload(ProjectModel.backends)) .execution_options(populate_existing=True) ) if lock_instances: @@ -339,6 +341,7 @@ async def select_run_candidate_fleet_models_with_filters( ) ) .options(raiseload(FleetModel.instances)) + .options(contains_eager(FleetModel.project).selectinload(ProjectModel.backends)) .execution_options(populate_existing=True) ) fleet_models_without_instances = list(res.unique().scalars().all()) @@ -424,7 +427,7 @@ async def find_optimal_fleet_with_offers( # Limit multinode runs to cluster fleets to guarantee best connectivity. continue - if not _run_can_fit_into_fleet(project, run_spec, fleet_model, fleet_spec): + if not _run_can_fit_into_fleet(run_spec, fleet_model, fleet_spec): logger.debug( "Skipping fleet %s from consideration: run cannot fit into fleet", fleet_model.name, @@ -727,7 +730,7 @@ def _get_required_instance_offers(run_spec: RunSpec, job: Job) -> int: def _run_can_fit_into_fleet( - project: ProjectModel, run_spec: RunSpec, fleet_model: FleetModel, fleet_spec: FleetSpec + run_spec: RunSpec, fleet_model: FleetModel, fleet_spec: FleetSpec ) -> bool: """ Returns `False` if the run cannot fit into fleet for sure. @@ -738,8 +741,8 @@ def _run_can_fit_into_fleet( and eventually removed by the fleet consolidation logic. """ nodes_required_num = get_nodes_required_num(run_spec) - if fleet_spec.configuration.ssh_config is not None or fleet_model.project_id != project.id: - # SSH fleets and imported cloud fleets cannot provision new instances for the run, + if fleet_spec.configuration.ssh_config is not None: + # SSH fleets cannot provision new instances for the run, # so only idle blocks of existing instances count. # Currently assume that each idle block can run a job. # TODO: Take resources / eligible offers into account. @@ -783,9 +786,7 @@ async def _get_backend_offers_in_fleet( if fleet_spec is None: fleet_spec = get_fleet_spec(fleet_model) try: - check_can_create_new_cloud_instance_in_fleet( - fleet_model, fleet_spec, job_project_id=project.id - ) + check_can_create_new_cloud_instance_in_fleet(fleet_model, fleet_spec) profile, requirements = get_run_profile_and_requirements_in_fleet( job=job, run_spec=run_spec, @@ -802,7 +803,9 @@ async def _get_backend_offers_in_fleet( # Handle multinode for old jobs that don't have requirements.multinode set. # TODO: Drop multinode param. multinode = requirements.multinode or is_multinode_job(job) - async with backends_cache.lease_project_backends(project) as backends: + # Offers come from the fleet's project backends, which differ from + # the run's project backends if the fleet is imported. + async with backends_cache.lease_project_backends(fleet_model.project) as backends: backends_with_offers = await get_offers_by_requirements( backends=backends, profile=profile, @@ -812,6 +815,9 @@ async def _get_backend_offers_in_fleet( volumes=volumes, privileged=job.job_spec.privileged, instance_mounts=check_run_spec_requires_instance_mounts(run_spec), + create_instance_support=new_instances_in_fleet_require_create_instance_support( + fleet_model, job_project_id=project.id + ), max_offers=max_offers, full_offers=full_offers, unallocated_resources=unallocated_resources, diff --git a/src/dstack/_internal/server/services/ssh.py b/src/dstack/_internal/server/services/ssh.py index 3a14b97d72..dfac77879d 100644 --- a/src/dstack/_internal/server/services/ssh.py +++ b/src/dstack/_internal/server/services/ssh.py @@ -71,8 +71,9 @@ def get_container_ssh_credentials( # As of 2026-03-13, the only container-based backend with SSH proxy is Kubernetes, # which is implemented as follows: the jump pod (JobProvisioningData.ssh_proxy) # is created once per project via Compute.run_job() with a public key submitted as - # a method argument, that is, with the public key of the project of the first (within - # that project) job submitted to the cluster. + # a method argument, that is, with the public key of the fleet's project. It is the + # job's project, since container-based backends are not used for importers' jobs + # (see `new_instances_in_fleet_require_create_instance_support()`). hosts.append((jpd.ssh_proxy, job_project_key)) target_host = SSHConnectionParams( hostname=jpd.hostname, diff --git a/src/tests/_internal/core/backends/daytona/test_compute.py b/src/tests/_internal/core/backends/daytona/test_compute.py index f68989ae0b..8fcefedcff 100644 --- a/src/tests/_internal/core/backends/daytona/test_compute.py +++ b/src/tests/_internal/core/backends/daytona/test_compute.py @@ -137,6 +137,7 @@ def _run_job(compute, offer=None, instance_name="dstack-test-unique", **override "run": SimpleNamespace(), "job": SimpleNamespace(job_spec=job_spec), "instance_offer": offer or _offers(compute, _item())[0], + "project_name": "main", "project_ssh_public_key": PUBLIC_KEY, "project_ssh_private_key": "private-key", "volumes": [], diff --git a/src/tests/_internal/core/backends/lium/test_compute.py b/src/tests/_internal/core/backends/lium/test_compute.py index 2425f7f27c..2ca1a7ae07 100644 --- a/src/tests/_internal/core/backends/lium/test_compute.py +++ b/src/tests/_internal/core/backends/lium/test_compute.py @@ -118,6 +118,7 @@ def _run_job(compute, **overrides): "run": SimpleNamespace(), "job": SimpleNamespace(job_spec=job_spec), "instance_offer": _offer(compute), + "project_name": "main", "project_ssh_public_key": PUBLIC_KEY, "project_ssh_private_key": "private-key", "volumes": [], diff --git a/src/tests/_internal/core/backends/runpod/test_compute.py b/src/tests/_internal/core/backends/runpod/test_compute.py index b573005891..8173e82ff3 100644 --- a/src/tests/_internal/core/backends/runpod/test_compute.py +++ b/src/tests/_internal/core/backends/runpod/test_compute.py @@ -62,7 +62,6 @@ def _job() -> MagicMock: def _run() -> MagicMock: run = MagicMock() - run.project_name = "main" run.user = "test-user" return run @@ -80,6 +79,7 @@ def _run_job( run=_run(), job=job if job is not None else _job(), instance_offer=offer, + project_name="main", project_ssh_public_key=PROJECT_SSH_PUBLIC_KEY, project_ssh_private_key="private-key", volumes=[], @@ -104,6 +104,7 @@ def _run_jobs(compute: RunpodCompute, offer: InstanceOfferWithAvailability, job_ run=_run(), job_configurations=job_configurations, instance_offer=offer, + project_name="main", project_ssh_public_key=PROJECT_SSH_PUBLIC_KEY, project_ssh_private_key="private-key", placement_group=None, diff --git a/src/tests/_internal/core/backends/vastai/test_compute.py b/src/tests/_internal/core/backends/vastai/test_compute.py index 3c6b1c71fe..65b4dff3b3 100644 --- a/src/tests/_internal/core/backends/vastai/test_compute.py +++ b/src/tests/_internal/core/backends/vastai/test_compute.py @@ -72,6 +72,7 @@ def _run_job(compute: VastAICompute, offer: InstanceOfferWithAvailability): run=run, job=job, instance_offer=offer, + project_name="main", project_ssh_public_key=PROJECT_SSH_PUBLIC_KEY, project_ssh_private_key="private-key", volumes=[], diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py index 3d281939e4..08f1ecd5f3 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_jobs_submitted.py @@ -818,7 +818,7 @@ async def test_creates_placement_group_for_cluster_fleet( assert fleet.lock_expires_at is None compute_mock.create_placement_group.assert_called_once() compute_mock.run_job.assert_called_once() - assert isinstance(compute_mock.run_job.call_args[0][6], PlacementGroup) + assert isinstance(compute_mock.run_job.call_args[0][7], PlacementGroup) placement_group = (await session.execute(select(PlacementGroupModel))).scalar() assert placement_group is not None @@ -877,11 +877,11 @@ async def test_creates_placement_group_in_reservation_only_if_compatible( placement_group = (await session.execute(select(PlacementGroupModel))).scalar() if compatible: compute_mock.create_placement_group.assert_called_once() - assert isinstance(compute_mock.run_job.call_args[0][6], PlacementGroup) + assert isinstance(compute_mock.run_job.call_args[0][7], PlacementGroup) assert placement_group is not None else: compute_mock.create_placement_group.assert_not_called() - assert compute_mock.run_job.call_args[0][6] is None + assert compute_mock.run_job.call_args[0][7] is None assert placement_group is None async def test_marks_unused_existing_placement_groups_for_cleanup( @@ -1822,17 +1822,19 @@ async def test_assigns_job_to_imported_fleet( assert job.fleet_id == fleet.id @pytest.mark.parametrize( - ("imported_idle_instances", "expected_fleet"), + ("imported_fleet_type", "imported_idle_instances", "expected_fleet"), [ - pytest.param(1, "own", id="imported-fleet-too-small"), - pytest.param(2, "imported", id="imported-fleet-fits"), + pytest.param("ssh", 1, "own", id="imported-ssh-fleet-too-small"), + pytest.param("ssh", 2, "imported", id="imported-ssh-fleet-fits"), + pytest.param("cloud", 1, "imported", id="imported-cloud-fleet-can-provision"), ], ) - async def test_selects_imported_cloud_fleet_only_if_idle_instances_fit_run( + async def test_selects_imported_fleet_only_if_run_fits( self, test_db, session: AsyncSession, worker: JobSubmittedWorker, + imported_fleet_type: str, imported_idle_instances: int, expected_fleet: str, ): @@ -1844,8 +1846,11 @@ async def test_selects_imported_cloud_fleet_only_if_idle_instances_fit_run( session=session, name="importer-project", owner=user ) repo = await create_repo(session=session, project_id=importer_project.id) - imported_fleet_spec = get_fleet_spec() - imported_fleet_spec.configuration.nodes = FleetNodesSpec(min=0, target=0, max=4) + if imported_fleet_type == "cloud": + imported_fleet_spec = get_fleet_spec() + imported_fleet_spec.configuration.nodes = FleetNodesSpec(min=0, target=0, max=4) + else: + imported_fleet_spec = get_fleet_spec(get_ssh_fleet_configuration()) imported_fleet = await create_fleet( session=session, project=exporter_project, spec=imported_fleet_spec ) @@ -1899,15 +1904,23 @@ async def test_selects_imported_cloud_fleet_only_if_idle_instances_fit_run( else: assert job.fleet_id == imported_fleet.id - async def test_does_not_provision_new_capacity_in_imported_cloud_fleet( + async def test_provisions_new_capacity_in_imported_cloud_fleet( self, test_db, session: AsyncSession, worker: JobSubmittedWorker ): user = await create_user(session=session) exporter_project = await create_project( - session=session, name="exporter-project", owner=user + session=session, + name="exporter-project", + owner=user, + ssh_public_key="exporter-public-key", + ssh_private_key="exporter-private-key", ) importer_project = await create_project( - session=session, name="importer-project", owner=user + session=session, + name="importer-project", + owner=user, + ssh_public_key="importer-public-key", + ssh_private_key="importer-private-key", ) repo = await create_repo(session=session, project_id=importer_project.id) fleet_spec = get_fleet_spec() @@ -1930,28 +1943,93 @@ async def test_does_not_provision_new_capacity_in_imported_cloud_fleet( offer = get_instance_offer_with_availability(backend=BackendType.AWS) backend_mock = Mock() + lease_project_backends_mock = mock_lease([backend_mock]) with patch( "dstack._internal.server.services.backends.cache.lease_project_backends", - mock_lease([backend_mock]), + lease_project_backends_mock, ): backend_mock.TYPE = BackendType.AWS backend_mock.compute.return_value.get_offers.return_value = [offer] + backend_mock.compute.return_value.run_job.return_value = get_job_provisioning_data( + dockerized=True, + backend=BackendType.AWS, + ) await _process_job(session=session, worker=worker, job_model=job) - backend_mock.compute.return_value.get_offers.assert_not_called() + job = await _get_job(session, job.id) + assert job.status == JobStatus.SUBMITTED + assert job.fleet_id == fleet.id + assert job.instance is not None + placeholder_id = job.instance.id + assert job.instance.project_id == exporter_project.id + res = await session.execute( + select(ProviderRefModel).where( + ProviderRefModel.provider_ref == job.instance.provider_ref + ) + ) + assert res.scalar_one().project_id == exporter_project.id + + await _process_job(session=session, worker=worker, job_model=job) + + # Offers and provisioning use the exporter's backends + assert lease_project_backends_mock.call_count > 0 + for lease_call in lease_project_backends_mock.call_args_list: + assert lease_call.args[0].id == exporter_project.id + job = await _get_job(session, job.id) + assert job.status == JobStatus.PROVISIONING + assert job.instance is not None + assert job.instance.id == placeholder_id + assert job.instance.project_id == exporter_project.id + assert job.instance.status == InstanceStatus.PROVISIONING + run_job_args = backend_mock.compute.return_value.run_job.call_args.args + assert run_job_args[3] == exporter_project.name + assert run_job_args[4] == exporter_project.ssh_public_key + assert run_job_args[5] == exporter_project.ssh_private_key + assert importer_project.ssh_public_key not in run_job_args[9] + assert exporter_project.ssh_public_key not in run_job_args[9] + + async def test_does_not_provision_new_capacity_in_imported_fleet_with_container_backend( + self, test_db, session: AsyncSession, worker: JobSubmittedWorker + ): + user = await create_user(session=session) + exporter_project = await create_project( + session=session, name="exporter-project", owner=user + ) + importer_project = await create_project( + session=session, name="importer-project", owner=user + ) + repo = await create_repo(session=session, project_id=importer_project.id) + fleet_spec = get_fleet_spec() + fleet_spec.configuration.nodes = FleetNodesSpec(min=0, target=0, max=2) + fleet = await create_fleet(session=session, project=exporter_project, spec=fleet_spec) + await create_export( + session=session, + exporter_project=exporter_project, + importer_projects=[importer_project], + exported_fleets=[fleet], + ) + run = await create_run(session=session, project=importer_project, repo=repo, user=user) + job = await create_job(session=session, run=run) + + offer = get_instance_offer_with_availability(backend=BackendType.KUBERNETES) + backend_mock = Mock() + backend_mock.TYPE = BackendType.KUBERNETES + backend_mock.compute.return_value.get_offers.return_value = [offer] + with patch( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([backend_mock]), + ): + # The first pass assigns the job to the fleet, the second one tries to provision + await _process_job(session=session, worker=worker, job_model=job) + job = await _get_job(session, job.id) + await _process_job(session=session, worker=worker, job_model=job) + job = await _get_job(session, job.id) assert job.status == JobStatus.TERMINATING assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY - assert job.termination_reason_message == "Fleet is at capacity" - assert not job.instance_assigned - res = await session.execute( - select(InstanceModel).where( - InstanceModel.fleet_id == fleet.id, - InstanceModel.deleted == False, - ) - ) - assert len(res.scalars().all()) == 1 + backend_mock.compute.return_value.get_offers.assert_not_called() + backend_mock.compute.return_value.run_job.assert_not_called() @pytest.mark.parametrize( "run_pinned_to_fleet", @@ -1985,7 +2063,9 @@ async def test_does_not_assign_job_with_volumes_to_imported_fleet( backend=BackendType.AWS, region="us-east-1", ) - fleet = await create_fleet(session=session, project=exporter_project) + fleet_spec = get_fleet_spec() + fleet_spec.configuration.nodes = FleetNodesSpec(min=0, target=0, max=None) + fleet = await create_fleet(session=session, project=exporter_project, spec=fleet_spec) await create_instance( session=session, project=exporter_project, @@ -2018,6 +2098,14 @@ async def test_does_not_assign_job_with_volumes_to_imported_fleet( assert job.status == JobStatus.TERMINATING assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY assert not job.instance_assigned + if run_pinned_to_fleet: + assert job.termination_reason_message == ( + "Cannot provision instances for jobs with volumes in imported fleets" + ) + res = await session.execute( + select(InstanceModel).where(InstanceModel.fleet_id == fleet.id) + ) + assert len(res.scalars().all()) == 1 async def test_assigns_job_to_specific_fleet( self, test_db, session: AsyncSession, worker: JobSubmittedWorker diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py index 8b7f4892e0..554e317b77 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_orphan_cleanup.py @@ -42,6 +42,7 @@ ) from dstack._internal.server.testing.common import ( create_backend, + create_fleet, create_instance, create_job, create_project, @@ -449,7 +450,13 @@ async def test_reports_orphans_and_marks_backend_clean( @pytest.mark.parametrize( "activity", - ["provider_ref_issued", "instance_started", "instance_finished", "job_finished"], + [ + "provider_ref_issued", + "instance_started", + "instance_finished", + "job_finished", + "job_finished_in_imported_fleet", + ], ) async def test_cleans_up_backend_with_activity_after_clean_at( self, test_db, session: AsyncSession, worker, activity: str @@ -466,12 +473,17 @@ async def test_cleans_up_backend_with_activity_after_clean_at( clean_at=clean_at, clean_mode=settings.ORPHAN_CLEANUP_MODE, ) - if activity == "job_finished": - repo = await create_repo(session=session, project_id=project.id) - run = await create_run(session=session, project=project, repo=repo, user=user) + if activity in ("job_finished", "job_finished_in_imported_fleet"): + fleet = await create_fleet(session=session, project=project) + run_project = project + if activity == "job_finished_in_imported_fleet": + run_project = await create_project(session=session, owner=user, name="importer") + repo = await create_repo(session=session, project_id=run_project.id) + run = await create_run(session=session, project=run_project, repo=repo, user=user) await create_job( session=session, run=run, + fleet=fleet, status=JobStatus.FAILED, last_processed_at=after_clean_at, ) @@ -517,6 +529,18 @@ async def test_skips_listing_without_activity_after_clean_at( InstanceStatus.IDLE, created_at=clean_at - timedelta(minutes=10), ) + # The project's job in a fleet imported from another project does not use its backends + exporter_project = await create_project(session=session, owner=user, name="exporter") + imported_fleet = await create_fleet(session=session, project=exporter_project) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run(session=session, project=project, repo=repo, user=user) + await create_job( + session=session, + run=run, + fleet=imported_fleet, + status=JobStatus.FAILED, + last_processed_at=clean_at + timedelta(minutes=10), + ) compute = _ListingCompute() item = await _lock(session, orphan_cleanup) with _patch_compute(compute): diff --git a/src/tests/_internal/server/services/runs/test_plan.py b/src/tests/_internal/server/services/runs/test_plan.py index 04b07805f1..1f5d85b14a 100644 --- a/src/tests/_internal/server/services/runs/test_plan.py +++ b/src/tests/_internal/server/services/runs/test_plan.py @@ -871,7 +871,7 @@ async def test_skips_backend_offers_when_instances_specified( class TestGetBackendOffersInFleet: @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - async def test_returns_no_offers_for_imported_fleet( + async def test_returns_offers_from_fleet_project_backends_for_imported_fleet( self, test_db, session: AsyncSession, monkeypatch: pytest.MonkeyPatch ) -> None: user = await create_user(session=session) @@ -896,11 +896,21 @@ async def test_returns_no_offers_for_imported_fleet( configuration=TaskConfiguration(image="debian", commands=["echo"]), ) jobs = await get_jobs_from_run_spec(run_spec=run_spec, secrets={}, replica_num=0) - get_offers_by_requirements_mock = AsyncMock() + offer = get_instance_offer_with_availability() + vm_backend = AsyncMock() + vm_backend.TYPE = BackendType.AWS + container_backend = AsyncMock() + container_backend.TYPE = BackendType.KUBERNETES + get_offers_by_requirements_mock = AsyncMock(return_value=[(vm_backend, offer)]) monkeypatch.setattr( "dstack._internal.server.services.runs.plan.get_offers_by_requirements", get_offers_by_requirements_mock, ) + lease_project_backends_mock = mock_lease([vm_backend, container_backend]) + monkeypatch.setattr( + "dstack._internal.server.services.backends.cache.lease_project_backends", + lease_project_backends_mock, + ) offers = await _get_backend_offers_in_fleet( project=importer_project, @@ -910,8 +920,55 @@ async def test_returns_no_offers_for_imported_fleet( volumes=None, ) - assert offers == [] - get_offers_by_requirements_mock.assert_not_awaited() + assert offers == [offer] + lease_project_backends_mock.assert_called_once() + assert lease_project_backends_mock.call_args.args[0].id == exporter_project.id + call_kwargs = get_offers_by_requirements_mock.call_args.kwargs + assert call_kwargs["backends"] == [vm_backend, container_backend] + # Container-based backends are not used for importers' runs + assert call_kwargs["create_instance_support"] is True + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_offers_from_container_backends_for_own_fleet( + self, test_db, session: AsyncSession, monkeypatch: pytest.MonkeyPatch + ) -> None: + user = await create_user(session=session) + project = await create_project(session=session, owner=user) + repo = await create_repo(session=session, project_id=project.id) + fleet_spec = get_fleet_spec() + fleet_spec.configuration.nodes = FleetNodesSpec(min=0, target=0, max=2) + fleet = await create_fleet(session=session, project=project, spec=fleet_spec) + run_spec = get_run_spec( + repo_id=repo.name, + configuration=TaskConfiguration(image="debian", commands=["echo"]), + ) + jobs = await get_jobs_from_run_spec(run_spec=run_spec, secrets={}, replica_num=0) + offer = get_instance_offer_with_availability(backend=BackendType.KUBERNETES) + container_backend = AsyncMock() + container_backend.TYPE = BackendType.KUBERNETES + get_offers_by_requirements_mock = AsyncMock(return_value=[(container_backend, offer)]) + monkeypatch.setattr( + "dstack._internal.server.services.runs.plan.get_offers_by_requirements", + get_offers_by_requirements_mock, + ) + monkeypatch.setattr( + "dstack._internal.server.services.backends.cache.lease_project_backends", + mock_lease([container_backend]), + ) + + offers = await _get_backend_offers_in_fleet( + project=project, + fleet_model=fleet, + run_spec=run_spec, + job=jobs[0], + volumes=None, + ) + + assert offers == [offer] + call_kwargs = get_offers_by_requirements_mock.call_args.kwargs + assert call_kwargs["backends"] == [container_backend] + assert call_kwargs["create_instance_support"] is False @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) diff --git a/src/tests/_internal/server/services/test_offers.py b/src/tests/_internal/server/services/test_offers.py index 02658c50e4..ca5d26199e 100644 --- a/src/tests/_internal/server/services/test_offers.py +++ b/src/tests/_internal/server/services/test_offers.py @@ -56,6 +56,37 @@ async def test_returns_multinode_offers(self): ) assert res == [(aws_backend_mock, aws_offer)] + @pytest.mark.asyncio + async def test_returns_create_instance_support_offers(self): + profile = Profile(name="test") + requirements = Requirements(resources=ResourcesSpec()) + aws_backend_mock = Mock() + aws_backend_mock.TYPE = BackendType.AWS + aws_offer = get_instance_offer_with_availability(backend=BackendType.AWS, price=1) + aws_backend_mock.compute.return_value.get_offers.return_value = [aws_offer] + runpod_backend_mock = Mock() + runpod_backend_mock.TYPE = BackendType.RUNPOD + runpod_offer = get_instance_offer_with_availability(backend=BackendType.RUNPOD, price=2) + runpod_backend_mock.compute.return_value.get_offers.return_value = [runpod_offer] + # Wrapper backends are kept, and their offers are filtered by base backend. + dstack_backend_mock = Mock() + dstack_backend_mock.TYPE = BackendType.DSTACK + dstack_gcp_offer = get_instance_offer_with_availability(backend=BackendType.GCP, price=3) + dstack_vastai_offer = get_instance_offer_with_availability( + backend=BackendType.VASTAI, price=4 + ) + dstack_backend_mock.compute.return_value.get_offers.return_value = [ + dstack_gcp_offer, + dstack_vastai_offer, + ] + res = await get_offers_by_requirements( + backends=[aws_backend_mock, runpod_backend_mock, dstack_backend_mock], + profile=profile, + requirements=requirements, + create_instance_support=True, + ) + assert res == [(aws_backend_mock, aws_offer), (dstack_backend_mock, dstack_gcp_offer)] + @pytest.mark.asyncio async def test_returns_volume_offers(self): profile = Profile(name="test")