From 0ac666b59708d11efb434ae7f59a9e0afd125c82 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Thu, 5 Nov 2026 13:37:40 +0545 Subject: [PATCH 01/12] Add concurrency sweeps to presets --- mkdocs/docs/concepts/presets.md | 6 +- .../_internal/cli/models/preset_agent.py | 15 +- .../_internal/cli/services/presets/build.py | 2 + .../_internal/cli/services/presets/output.py | 62 +++++-- .../presets/resources/system_prompt.md | 65 +++++--- .../_internal/cli/services/presets/session.py | 93 +++++++++-- .../_internal/cli/services/presets/verify.py | 30 +++- .../cli/services/presets/workspace.py | 51 +++++- .../_internal/core/models/configurations.py | 53 +++++- src/dstack/_internal/core/models/presets.py | 37 ++++- src/tests/_internal/cli/common.py | 3 +- .../cli/services/presets/test_agent.py | 129 +++++++++++++++ .../cli/services/presets/test_create.py | 6 +- .../cli/services/presets/test_output.py | 155 +++++++++++++++++- .../services/presets/test_report_schema.py | 67 ++++---- .../cli/services/presets/test_store.py | 4 +- .../cli/services/presets/test_verify.py | 134 +++++++++++++-- .../cli/services/presets/test_workspace.py | 59 +++++++ .../core/models/test_configurations.py | 43 +++++ .../_internal/core/models/test_presets.py | 30 ++++ 20 files changed, 911 insertions(+), 133 deletions(-) diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index b01c63f62b..4096f11226 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -33,8 +33,9 @@ trials: 5 min_context_length: 1048576 max_ttft: 675 -# The number of simultaneous requests every benchmark uses -concurrency: 1 +# The numbers of simultaneous requests every benchmark is run at: +# a value, a list, `2,3`, or a doubling range `2..16` +concurrency: 2..16 # The request shape every benchmark uses (defaults to 1024 and 1024) input_tokens: 10000 @@ -390,7 +391,6 @@ At the same time, it's recommended to create presets using your own agent — ei ## Limitations * Currently, the agent doesn't upload compiled binaries anywhere; patches compile at runtime -* Doesn't support ranges for `concurrency` > Report bugs and request features on [GitHub](https://github.com/dstackai/dstack/issues), and ask questions on [Discord](https://discord.gg/u8SmfwPpMd). diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 812ddce17e..918e622737 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -9,6 +9,7 @@ ValidationError, ValidatorFunctionWrapHandler, WrapValidator, + model_validator, ) from dstack._internal.core.models.common import CoreModel @@ -53,7 +54,19 @@ class PresetAgentSuccess(CoreModel): base: Annotated[str, Field(min_length=1)] model: Annotated[str, Field(min_length=1)] context_length: PositiveInt - benchmark: PresetBenchmark + benchmarks: Annotated[list[PresetBenchmark], Field(min_length=1)] + + @model_validator(mode="before") + @classmethod + def wrap_legacy_single_benchmark(cls, data: Any) -> Any: + # An agent process started by a pre-sweep CLI reports one `benchmark`, and + # a session detached before the upgrade is attached and verified by the + # new CLI. The one point is its whole sweep; the sweep check in `verify` + # still requires it to be the requested concurrency. + if isinstance(data, dict) and "benchmark" in data and "benchmarks" not in data: + data = dict(data) + data["benchmarks"] = [data.pop("benchmark")] + return data class PresetAgentFailure(CoreModel): diff --git a/src/dstack/_internal/cli/services/presets/build.py b/src/dstack/_internal/cli/services/presets/build.py index 1c3137f7aa..1fa4971158 100644 --- a/src/dstack/_internal/cli/services/presets/build.py +++ b/src/dstack/_internal/cli/services/presets/build.py @@ -32,6 +32,7 @@ def build_preset( base_model: str, repo: str, context_length: int, + benchmarks: list[PresetBenchmark], benchmark: PresetBenchmark, configuration: PresetConfiguration, best_trial: int, @@ -53,6 +54,7 @@ def build_preset( configuration=configuration, created_at=created_at, service=service, + benchmarks=benchmarks, benchmark=benchmark, verified_on=verification_replica_groups, ) diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 86f39d54c7..f5cf9fde07 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Any, Optional, Sequence +from typing import Any, Callable, Optional, Sequence from rich.table import Table @@ -198,9 +198,9 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False share = round(100 * shared_prefix_tokens / input_tokens) if share: objective.append(f"prefix={share}%") - concurrency = (best or {}).get("concurrency") or constraints.get("concurrency") - if concurrency: - objective.append(f"c={concurrency}") + sweep = constraints.get("concurrency") or (best or {}).get("concurrency") + if sweep: + objective.append(f"c={_format_concurrency(sweep)}") min_context_length = constraints.get("min_context_length") if verbose and isinstance(min_context_length, int): objective.append(f"ctx>={_format_token_count(min_context_length)}") @@ -211,14 +211,18 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False tps = _format_number(best["tok_s"]) if objective: # Lead with per-user tps: comparable across rows regardless of concurrency. - tpot_ms = best.get("tpot_ms") - if isinstance(tpot_ms, (int, float)) and tpot_ms > 0: - parts.append(f"tps/user={_format_number(1000 / tpot_ms)}") + per_user = best.get("per_user_tok_s_range") + if per_user: + parts.append(f"tps/user={_format_range(per_user, _format_number)}") + # Which point of a sweep the numbers are from; redundant for one value. + if isinstance(sweep, list) and len(sweep) > 1 and best.get("concurrency"): + parts.append(f"c={best['concurrency']}") if verbose: - parts.append(f"tps={tps}") - ttft_ms = best.get("ttft_ms") - if isinstance(ttft_ms, (int, float)): - parts.append(f"ttft={_format_duration_ms(ttft_ms)}") + tok_s_range = best.get("tok_s_range") or [best["tok_s"]] + parts.append(f"tps={_format_range(tok_s_range, _format_number)}") + ttft_range = best.get("ttft_ms_range") + if ttft_range: + parts.append(f"ttft={_format_range(ttft_range, _format_duration_ms)}") context_length = best.get("context_length") if isinstance(context_length, int): parts.append(f"ctx={_format_token_count(context_length)}") @@ -304,7 +308,7 @@ def format_preset_objective( share = round(100 * workload.shared_prefix_tokens / workload.input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={workload.concurrency}") + parts.append(f"c={_format_concurrency([b.workload.concurrency for b in preset.benchmarks])}") return f"[secondary]{' '.join(parts)}[/]" @@ -323,7 +327,7 @@ def _format_creation_objective( share = round(100 * (configuration.shared_prefix_tokens or 0) / input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={configuration.concurrency or workload.concurrency}") + parts.append(f"c={_format_concurrency(configuration.concurrency or [workload.concurrency])}") if verbose and configuration.min_context_length is not None: parts.append(f"ctx>={_format_token_count(configuration.min_context_length)}") if verbose and configuration.max_ttft is not None: @@ -345,15 +349,22 @@ def _breaches_constraints(preset: AnyStoredPreset) -> bool: def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) -> str: - benchmark = preset.benchmark - metrics = benchmark.metrics + # A sweep shows each metric as its range across concurrencies, and `c=` names + # the selected point. + benchmarks = preset.benchmarks parts = [ - f"tps/user={_format_number(benchmark.effective_per_user_tok_per_s)}", + "tps/user=" + + _format_range([b.effective_per_user_tok_per_s for b in benchmarks], _format_number) ] + if len(benchmarks) > 1: + parts.append(f"c={preset.benchmark.workload.concurrency}") if verbose: - parts.append(f"tps={_format_number(benchmark.effective_output_tok_per_s)}") + parts.append( + "tps=" + + _format_range([b.effective_output_tok_per_s for b in benchmarks], _format_number) + ) parts += [ - f"ttft={_format_duration_ms(metrics.ttft_ms.p50)}", + "ttft=" + _format_range([b.metrics.ttft_ms.p50 for b in benchmarks], _format_duration_ms), f"ctx={_format_token_count(preset.context_length)}", ] text = " ".join(parts) @@ -362,6 +373,21 @@ def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) - return text +def _format_range(values: list[float], fmt: Callable[[float], str]) -> str: + low, high = fmt(min(values)), fmt(max(values)) + return low if low == high else f"{low}..{high}" + + +def _format_concurrency(value: Any) -> str: + """A sweep in the shape the user wrote it: `8`, `2,3`, or `2..16` when every + step doubles. A pre-sweep record carries a bare int.""" + if not isinstance(value, list): + return str(value) + if len(value) > 1 and all(b == a * 2 for a, b in zip(value, value[1:])): + return f"{value[0]}..{value[-1]}" + return ",".join(str(v) for v in value) + + def _format_duration_ms(value: float) -> str: # 999.6 rounds to 1000, which must read as 1s rather than 1000ms. if value < 999.5: diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 691df34b33..79605eeec3 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -5,9 +5,21 @@ hardware, through sequential experimental trials. ## Performance -Performance is `output_tok_per_s`, the output token throughput of a benchmark -run at the `concurrency` from `constraints.json` (see `## Benchmark`). Benchmark -tools report several throughput numbers; performance is the output one. +Performance is the highest `output_tok_per_s` a configuration reaches across +the `concurrency` sweep from `constraints.json` (see `## Benchmark`), counting +only sweep points that meet `max_ttft`, the maximum p50 time to first token, in +milliseconds (see `# Constraints`). Points that breach `max_ttft` are ignored. +For example, with `max_ttft` of 2000: + +| `concurrency` | `output_tok_per_s` | `ttft_ms.p50` | +|---|---|---| +| 4 | 130 | 400 | +| 8 | 240 | 1500 | +| 16 | 380 | 2400 | + +The point at 16 breaches `max_ttft`, so it is ignored. Performance is 240, the +highest throughput among the remaining points. Benchmark tools report several +throughput numbers; performance is the output one. Significantly better performance goes beyond what conventional configurations and published benchmarks reach; they are not the ceiling. Estimate the @@ -35,10 +47,9 @@ Field semantics: - `max_ttft`: the maximum p50 time to first token, in milliseconds, that any benchmark in this session may report. - `trials_num`: the number of trials in this session. -- `concurrency`: the number of simultaneous requests for every benchmark in - this session. It is fixed so that benchmark results are comparable. - +- `concurrency`: the numbers of simultaneous requests every benchmark in + this session is run at, one benchmark per value. The list is fixed so + that benchmark results are comparable across trials. - `dataset`: the benchmark dataset for every benchmark in this session; see `## Benchmark`. @@ -257,7 +268,7 @@ mindful of which specific change was the root cause. others: ``` -{"resources": {...}, "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} +{"resources": {...}, "context_length": ..., "benchmarks": [{...}, ...], "learned": ..., "failed": ...} ``` 2. In case the task is using node groups (a router group plus worker @@ -265,7 +276,7 @@ mindful of which specific change was the root cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} +{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": ..., "benchmarks": [{...}, ...], "learned": ..., "failed": ...} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -296,12 +307,14 @@ mindful of which specific change was the root cause. - `context_length`: the largest context the trial's configuration handles, found as described in `## Benchmark`; `null` only when the benchmark couldn't be done at all. -- `benchmark`: the trial benchmark (see `## Benchmark` for the structure); - `null` only when the benchmark couldn't be done at all. +- `benchmarks`: one trial benchmark per value of `concurrency`, in that + order (see `## Benchmark` for the structure); `null` only when no benchmark + could be done at all. - `learned`: the major things this trial taught you that you did not know before it ran. Required for every trial, including a failed one. - `failed`: `true` if the configuration never ran or the benchmark broke a - constraint such as `max_ttft` or `min_context_length`, absent otherwise. + constraint such as `min_context_length`, or no sweep point met `max_ttft`; + absent otherwise. Breaching `max_ttft` at some points only is not a failure. You're expected to do exactly `trials_num` trials (see `# Constraints`). @@ -317,9 +330,11 @@ no trials remain. In that case, log the failure to `final_report.json` (see ## Benchmark During trials, run benchmarks via SSH inside the task, directly against the -serving engine: use `dataset` and `concurrency``concurrency`, `input_tokens`, `output_tokens`, and -`shared_prefix_tokens` from `constraints.json` and measure all trials the same -way so that their results are comparable with each other. +serving engine: run one benchmark per value of `concurrency`, using +`dataset``input_tokens`, `output_tokens`, and +`shared_prefix_tokens` from `constraints.json`, and measure all trials the same +way so that their results are comparable with each other. Use a different +seed for each point of the sweep. In case the task is using a router (see `## Cluster Placement`), run benchmarks via SSH inside the router node, directly against the router engine, never against a @@ -375,9 +390,10 @@ never part of the measured metrics. All verification and benchmark requests must succeed. -Record every benchmark using the following structure and field names — -trial benchmarks in `trials//trial.json`, the final benchmark as -`final_report.json.benchmark` (values are illustrative): +Record every benchmark using the following structure and field names, one +object per value of `concurrency` — trial benchmarks in +`trials//trial.json.benchmarks`, the final benchmarks as +`final_report.json.benchmarks` (values are illustrative): ```json { @@ -407,9 +423,9 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free -invocation. For the final benchmark, run it with streaming responses, set -`workload.concurrency` to `concurrency` from `constraints.json`, produce -every field of the structure, and calculate all metrics from the +invocation. For the final benchmarks, run each with streaming responses, one +per value of `concurrency` from `constraints.json` with `workload.concurrency` +set to that value, produce every field of the structure, and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and warmup requests. Never invent missing values. @@ -594,12 +610,12 @@ variable listed under `env` in `constraints.json`. Do not put secret values in `final_report.json`, or print them. Use env references in `final_report.json.service_yaml`; use environment variable names or redacted values in -`final_report.json.benchmark.command`. +`final_report.json.benchmarks[].command`. # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmark`, +`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmarks`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed @@ -614,7 +630,8 @@ trial remained), include exactly: - `model`: the exact repo/path loaded by the final service command - `context_length`: the largest context verified for the final service, as described in `## Benchmark` -- `benchmark`: the final service benchmark described in `## Benchmark` +- `benchmarks`: the final service benchmarks described in `## Benchmark`, + one per value of `concurrency` Set `final_report.json.base` as follows: diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index 9ce77213e3..dc09cc2b5d 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -529,9 +529,12 @@ def list_preset_sessions() -> list[dict[str, Any]]: entry["id"] = path.name entry["name"] = claimed_session_name(state) entry["status"] = status - entry["trials"] = _summarize_session_trials(path / TRIALS_DIRNAME) + constraints = _read_session_constraints(path) + entry["trials"] = _summarize_session_trials( + path / TRIALS_DIRNAME, max_ttft=constraints.get("max_ttft") + ) entry["verification"] = _read_last_session_verification(path / SERVICE_DIRNAME) - entry["constraints"] = _read_session_constraints(path) + entry["constraints"] = constraints entries.append(entry) return entries @@ -584,7 +587,9 @@ def _read_last_session_verification(path: Path) -> Optional[dict[str, Any]]: return {"status": "verifying"} -def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: +def _summarize_session_trials( + path: Path, *, max_ttft: Optional[float] = None +) -> Optional[dict[str, Any]]: # TODO: Refactor this crap - must be explicit what is this and where is it used; also dicts are prohibited in dstack repo """A trial directory without `trial.json` is still in flight and is not counted.""" @@ -605,35 +610,25 @@ def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: gpu: Optional[str] = None for record in records: count += 1 - benchmark = record.get("benchmark") failed.append(bool(record.get("failed"))) record_gpu = _format_trial_gpu(record) if record_gpu: gpu = record_gpu - if not isinstance(benchmark, dict): - series.append(None) - continue - metrics = benchmark.get("metrics") or {} - workload = benchmark.get("workload") or {} - duration = metrics.get("duration_seconds") - tokens = metrics.get("total_output_tokens") - if not isinstance(duration, (int, float)) or duration <= 0: - series.append(None) - continue - if not isinstance(tokens, (int, float)): + point = _select_trial_point(record, max_ttft) + if point is None: series.append(None) continue - tok_s = tokens / duration + tok_s, metrics, workload, sweep = point series.append(tok_s) # A failed trial keeps its benchmark — it is what the next trial learns # from — but it is not a candidate for best, and promoting one would put # a configuration that broke a constraint at the top of the listing. if record.get("failed"): if best_failed is None or tok_s > best_failed["tok_s"]: - best_failed = _trial_entry(tok_s, record, metrics, workload, record_gpu) + best_failed = _trial_entry(tok_s, record, metrics, workload, sweep, record_gpu) continue if best is None or tok_s > best["tok_s"]: - best = _trial_entry(tok_s, record, metrics, workload, record_gpu) + best = _trial_entry(tok_s, record, metrics, workload, sweep, record_gpu) return { "count": count, "best": best, @@ -644,11 +639,55 @@ def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: } +def _select_trial_point( + record: dict[str, Any], max_ttft: Optional[float] +) -> Optional[tuple[float, dict[str, Any], dict[str, Any], list[dict[str, Any]]]]: + """The sweep point a trial is judged by, as `select_benchmark` picks it for a + preset: the highest throughput within `max_ttft`, else the closest TTFT. + A record from before the sweep holds a single `benchmark`. Returns + `(tok_s, metrics, workload, sweep)`, where `sweep` is the metrics of every + point with usable numbers, or `None` when there is none.""" + benchmarks = record.get("benchmarks") + if not isinstance(benchmarks, list): + benchmarks = [record.get("benchmark")] + points: list[tuple[bool, float, Any, dict[str, Any], dict[str, Any]]] = [] + for benchmark in benchmarks: + if not isinstance(benchmark, dict): + continue + metrics = benchmark.get("metrics") or {} + workload = benchmark.get("workload") or {} + duration = metrics.get("duration_seconds") + tokens = metrics.get("total_output_tokens") + if not isinstance(duration, (int, float)) or duration <= 0: + continue + if not isinstance(tokens, (int, float)): + continue + ttft = (metrics.get("ttft_ms") or {}).get("p50") + within = not isinstance(max_ttft, (int, float)) or ( + isinstance(ttft, (int, float)) and ttft <= max_ttft + ) + points.append((within, tokens / duration, ttft, metrics, workload)) + if not points: + return None + sweep = [p[3] for p in points] + passing = [p for p in points if p[0]] + if passing: + _, tok_s, _, metrics, workload = max(passing, key=lambda p: p[1]) + else: + # `ttft` may be missing in a raw record; sort those last. + _, tok_s, _, metrics, workload = min( + points, + key=lambda p: (p[2] if isinstance(p[2], (int, float)) else float("inf"), -p[1]), + ) + return tok_s, metrics, workload, sweep + + def _trial_entry( tok_s: float, record: dict[str, Any], metrics: dict[str, Any], workload: dict[str, Any], + sweep: list[dict[str, Any]], gpu: Optional[str], ) -> dict[str, Any]: ttft = (metrics.get("ttft_ms") or {}).get("p50") @@ -659,12 +698,30 @@ def _trial_entry( "tok_s": tok_s, "tpot_ms": tpot if isinstance(tpot, (int, float)) and tpot > 0 else None, "ttft_ms": ttft if isinstance(ttft, (int, float)) else None, + # The trial's spread across its sweep, as `[low, high]`, None when no + # point reports the value. One point gives `low == high`. + "tok_s_range": _range([m["total_output_tokens"] / m["duration_seconds"] for m in sweep]), + "per_user_tok_s_range": _range( + [1000 / v for v in (_latency(m, "tpot_ms", "mean") for m in sweep) if v] + ), + "ttft_ms_range": _range( + [v for v in (_latency(m, "ttft_ms", "p50") for m in sweep) if v is not None] + ), "context_length": context_length if isinstance(context_length, int) else None, "concurrency": workload.get("concurrency"), "gpu": gpu, } +def _latency(metrics: dict[str, Any], name: str, stat: str) -> Optional[float]: + value = (metrics.get(name) or {}).get(stat) + return value if isinstance(value, (int, float)) and value >= 0 else None + + +def _range(values: list[float]) -> Optional[list[float]]: + return [min(values), max(values)] if values else None + + def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: """A trial records its hardware in one of two formats, as described in `system_prompt.md`: `{"resources": {...}}` without node groups, and diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 643522c503..c473a5b4b4 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -31,6 +31,7 @@ PresetBenchmark, PresetVerificationReplicaGroup, PresetWorkload, + select_benchmark, ) from dstack._internal.core.models.runs import JobStatus, Run, RunStatus @@ -99,6 +100,10 @@ def build_verified_preset( _check_report_answers_request(report, preset_configuration) if service.model is None or service.model.name != preset_configuration.model.api_model_name: raise CLIError("The agent's final service model name does not match the requested model") + benchmarks = sorted( + (_normalized_benchmark(b, preset_configuration) for b in report.benchmarks), + key=lambda b: b.workload.concurrency, + ) return build_preset( name=name, service=_portable_service( @@ -114,7 +119,8 @@ def build_verified_preset( # it cannot be confused with the service's client-facing model name. repo=report.model, context_length=report.context_length, - benchmark=_normalized_benchmark(report.benchmark, preset_configuration), + benchmarks=benchmarks, + benchmark=select_benchmark(benchmarks, preset_configuration.max_ttft), best_trial=report.trial, configuration=preset_configuration, preset_id=preset_id, @@ -139,9 +145,11 @@ def _check_report_answers_request( report: PresetAgentSuccess, configuration: PresetConfiguration ) -> None: """The report must answer what the configuration asked: the same benchmark - workload, and the requested model — exactly when it was exact, any variant of - the base otherwise.""" - _check_workload_answers_request(report.benchmark.workload, configuration) + workload at every concurrency of the sweep, and the requested model — exactly + when it was exact, any variant of the base otherwise.""" + for benchmark in report.benchmarks: + _check_workload_answers_request(benchmark.workload, configuration) + _check_sweep_answers_request(report.benchmarks, configuration) if configuration.model.allows_variant_selection: if report.base != configuration.model.api_model_name: raise CLIError("The agent's final report base does not match the requested model") @@ -172,10 +180,18 @@ def _check_workload_answers_request( f"The agent's final benchmark shared prefix of {workload.shared_prefix_tokens}" f" tokens does not match the requested {shared_prefix_tokens}" ) - if configuration.concurrency is not None and workload.concurrency != configuration.concurrency: + + +def _check_sweep_answers_request( + benchmarks: Sequence[PresetBenchmark], configuration: PresetConfiguration +) -> None: + """One benchmark per requested concurrency, no more and no fewer: a missing + point would hide where the service stops meeting the constraints.""" + reported = sorted(b.workload.concurrency for b in benchmarks) + if configuration.concurrency is not None and reported != configuration.concurrency: raise CLIError( - f"The agent's final benchmark concurrency of {workload.concurrency} does not match the" - f" requested concurrency of {configuration.concurrency}" + f"The agent's final benchmark concurrencies {reported} do not match the" + f" requested {configuration.concurrency}" ) diff --git a/src/dstack/_internal/cli/services/presets/workspace.py b/src/dstack/_internal/cli/services/presets/workspace.py index 97ade98b38..7d5b609cf6 100644 --- a/src/dstack/_internal/cli/services/presets/workspace.py +++ b/src/dstack/_internal/cli/services/presets/workspace.py @@ -10,7 +10,7 @@ from contextlib import suppress from dataclasses import dataclass from pathlib import Path -from typing import Optional, Sequence +from typing import Any, Optional, Sequence from dstack._internal.cli.models.preset_agent import PresetSessionWorkspace from dstack._internal.cli.services.presets.session import ( @@ -300,7 +300,7 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: for name in (_CONSTRAINTS_FILENAME, _FINAL_REPORT_FILENAME): if (source_root / name).is_file(): target_root.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source_root / name, target_root / name) + _copy_record(source_root / name, target_root / name) copied = True for group, filenames in ( (TRIALS_DIRNAME, ("trial.json", "task.dstack.yml")), @@ -316,7 +316,7 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: for name in filenames: if (record_dir / name).is_file(): target_dir.mkdir(parents=True, exist_ok=True) - shutil.copyfile(record_dir / name, target_dir / name) + _copy_record(record_dir / name, target_dir / name) copied = True patches = record_dir / "patches" if group == TRIALS_DIRNAME and patches.is_dir(): @@ -325,6 +325,51 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: return copied +# The records an agent reads through `previous/`, upgraded on copy so a +# session recorded before concurrency sweeps reads in the format the agent +# writes. The originals stay as they were written. +_UPGRADED_RECORDS = (_CONSTRAINTS_FILENAME, _FINAL_REPORT_FILENAME, "trial.json") + + +def _copy_record(source: Path, target: Path) -> None: + if source.name not in _UPGRADED_RECORDS: + shutil.copyfile(source, target) + return + try: + data = json.loads(source.read_text(encoding="utf-8")) + except (OSError, ValueError): + # A torn or foreign file is still the agent's to read as written. + shutil.copyfile(source, target) + return + upgraded = _upgrade_pre_sweep_record(source.name, data) + if upgraded is data: + shutil.copyfile(source, target) + return + target.write_text(json.dumps(upgraded, indent=2) + "\n", encoding="utf-8") + + +def _upgrade_pre_sweep_record(name: str, data: Any) -> Any: + # Before the sweep, `constraints.json` held one `concurrency` and trial and + # final records held one `benchmark`: that one point is the whole sweep. + if not isinstance(data, dict): + return data + if name == _CONSTRAINTS_FILENAME: + concurrency = data.get("concurrency") + if isinstance(concurrency, int): + return {**data, "concurrency": [concurrency]} + return data + if "benchmark" not in data or "benchmarks" in data: + return data + upgraded = {} + for key, value in data.items(): + if key == "benchmark": + # `null` means no benchmark could be done; it stays `null`. + upgraded["benchmarks"] = None if value is None else [value] + else: + upgraded[key] = value + return upgraded + + def _install_skills(target_dir: Path) -> None: source_dir = _get_skills_dir() for skill_name in _SKILL_NAMES: diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 6cf8e423ff..bb2caaafa3 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -1774,12 +1774,14 @@ class PresetConfiguration( ), ] = None concurrency: Annotated[ - Optional[PositiveInt], + Optional[list[PositiveInt]], Field( + min_length=1, description=( - "The number of simultaneous requests used for benchmarks during preset" - " creation. Required for creation" - ) + "The numbers of simultaneous requests every benchmark is run at during" + " preset creation: a single value, a list, `2,3`, or a doubling range" + " `2..16` (2, 4, 8, 16). Required for creation" + ), ), ] = None input_tokens: Annotated[ @@ -1866,6 +1868,22 @@ def effective_output_tokens(self) -> int: def effective_baseline(self) -> bool: return self.baseline if self.baseline is not None else DEFAULT_BASELINE + @field_validator( + "concurrency", + mode="before", + json_schema_input_type=Union[PositiveInt, str, list[PositiveInt]], + ) + @classmethod + def parse_concurrency(cls, value: Any) -> Any: + return _parse_concurrency_sweep(value) + + @field_validator("concurrency") + @classmethod + def sort_concurrency(cls, value: Optional[list[int]]) -> Optional[list[int]]: + # Ascending and unique, whichever way it was written: the sweep is compared + # to what the agent reports, and the report is ordered the same way. + return sorted(set(value)) if value is not None else None + @field_validator("dataset") @classmethod def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]: @@ -1946,6 +1964,33 @@ def validate_prompt(cls, value: Any) -> Any: return value +def _parse_concurrency_sweep(value: Any) -> Any: + # `8`, `2,3`, `[2, 3]`, or `2..16`. A range doubles from its start and must + # land exactly on its end, so `3..16` is rejected rather than stopping at 12 + # or overshooting to 24. Element validation is left to the field type. + if value is None or isinstance(value, (bool, list)): + return value + if isinstance(value, int): + return [value] + if not isinstance(value, str): + return value + text = value.replace(" ", "") + if ".." not in text: + return text.split(",") + start_text, _, end_text = text.partition("..") + if not start_text.isdigit() or not end_text.isdigit(): + raise ValueError("a concurrency range needs both ends, e.g. `2..16`") + start, end = int(start_text), int(end_text) + if start < 1 or end < start: + raise ValueError("a concurrency range must go from a positive start up to its end") + values = [start] + while values[-1] < end: + values.append(values[-1] * 2) + if values[-1] != end: + raise ValueError(f"concurrency range end {end} is not reachable by doubling from {start}") + return values + + def _validate_model(value: Any, *, field: str) -> str: if not isinstance(value, str) or not value.strip(): raise ValueError(f"Preset model {field} must be a non-empty string") diff --git a/src/dstack/_internal/core/models/presets.py b/src/dstack/_internal/core/models/presets.py index 95e87fb29f..1258dcb1b1 100644 --- a/src/dstack/_internal/core/models/presets.py +++ b/src/dstack/_internal/core/models/presets.py @@ -128,6 +128,22 @@ def validate_metrics(self) -> Self: return self +def select_benchmark( + benchmarks: Sequence[PresetBenchmark], max_ttft: Optional[int] +) -> PresetBenchmark: + """Picks the sweep point a preset is judged by. + + Rule 1: Keep the points whose p50 TTFT is within `max_ttft`. + Rule 2: If any are kept, pick the one with the highest output throughput. + Rule 3: If none are kept, pick the one with the lowest p50 TTFT, so the + preset records how close it got to the cap rather than how fast it ran. + Rule 4: If two points tie on TTFT in rule 3, pick the higher throughput.""" + passing = [b for b in benchmarks if max_ttft is None or b.metrics.ttft_ms.p50 <= max_ttft] + if passing: + return max(passing, key=lambda b: b.effective_output_tok_per_s) + return min(benchmarks, key=lambda b: (b.metrics.ttft_ms.p50, -b.effective_output_tok_per_s)) + + class PresetVerificationReplicaGroup(CoreModel): # The service replica group this was measured for. name: str @@ -153,16 +169,35 @@ class PortablePreset(CoreModel): # stored as-is. `files` paths are stored relative to the preset directory, # absolute after load. service: ServiceConfiguration + benchmarks: Annotated[List[PresetBenchmark], Field(min_length=1)] + """The final service benchmarked at every concurrency of the sweep, ascending.""" benchmark: PresetBenchmark + """The one of `benchmarks` the preset is judged and listed by, chosen by + `select_benchmark`. Kept as its own field so clients that predate the sweep + keep reading it.""" # The hardware it was verified on: the actual resources of every running # replica, by service replica group. verified_on: List[PresetVerificationReplicaGroup] + @model_validator(mode="before") + @classmethod + def wrap_legacy_single_benchmark(cls, data: Any) -> Any: + # Before the sweep, a preset was benchmarked at one concurrency and + # stored it as `benchmark` alone. That one point is its whole sweep. + if isinstance(data, dict) and "benchmark" in data and "benchmarks" not in data: + data = {**data, "benchmarks": [data["benchmark"]]} + return data + @model_validator(mode="after") def validate_artifact(self) -> Self: service = self.service if service.model is None: raise ValueError("preset service must specify model") + concurrencies = [b.workload.concurrency for b in self.benchmarks] + if concurrencies != sorted(set(concurrencies)): + raise ValueError("preset benchmarks must be at unique, ascending concurrencies") + if self.benchmark not in self.benchmarks: + raise ValueError("preset benchmark must be one of its benchmarks") if any(group.resources is None for group in service.replica_groups): raise ValueError("preset service must specify resources") for field in PRESET_EXCLUDED_FIELDS: @@ -304,7 +339,7 @@ class PresetConstraints(CoreModel): min_context_length: PositiveInt max_ttft: PositiveInt trials_num: PositiveInt - concurrency: PositiveInt + concurrency: Annotated[list[PositiveInt], Field(min_length=1)] baseline: bool fleets: Annotated[list[str], Field(min_length=1)] env: list[str] diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index add3afc533..144d2e1a2b 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -171,6 +171,7 @@ def get_preset( ), context_length=context_length, best_trial=1, + benchmarks=[get_preset_benchmark()], benchmark=get_preset_benchmark(), verified_on=[ PresetVerificationReplicaGroup(name=DEFAULT_REPLICA_GROUP_NAME, replicas=[resources]) @@ -278,7 +279,7 @@ def get_successful_preset_report(run: Run) -> PresetAgentSuccess: "base": "Qwen/Qwen3.5-27B", "model": "community/Qwen3.5-27B-GPTQ-Int4", "context_length": 32768, - "benchmark": get_preset_benchmark(), + "benchmarks": [get_preset_benchmark()], } ) diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index e55c3fd013..4167331685 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -1642,6 +1642,9 @@ def test_counts_records_even_when_trials_share_a_task(self, tmp_path): "tok_s": 2300.0, "tpot_ms": None, "ttft_ms": None, + "tok_s_range": [2300.0, 2300.0], + "per_user_tok_s_range": None, + "ttft_ms_range": None, "context_length": None, "concurrency": 8, "gpu": "A40:48GB:1", @@ -1684,6 +1687,132 @@ def test_a_trial_still_in_flight_is_not_counted(self, tmp_path): assert summary["count"] == 1 assert summary["series"] == [2000.0] + def test_judges_a_sweep_by_its_fastest_point_within_max_ttft(self, tmp_path): + def point(concurrency, tok_s, ttft_p50): + return { + "workload": {"concurrency": concurrency}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": tok_s, + "ttft_ms": {"p50": ttft_p50}, + }, + } + + record = { + "resources": {}, + # Throughput rises with concurrency, but the top point breaches the cap. + "benchmarks": [point(2, 500.0, 100.0), point(4, 800.0, 300.0), point(8, 900.0, 900.0)], + } + trials_dir = _write_trials(tmp_path, [record]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + assert summary["series"] == [800.0] + best = summary["best"] + assert best is not None + assert best["concurrency"] == 4 + assert best["ttft_ms"] == 300.0 + # The selected point is c=4; the ranges still cover every point. + assert best["ttft_ms_range"] == [100.0, 900.0] + + def test_ranges_come_from_the_selected_trial_only(self, tmp_path): + def point(concurrency, tokens, ttft, tpot): + return { + "workload": {"concurrency": concurrency}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": tokens, + "ttft_ms": {"p50": ttft}, + "tpot_ms": {"mean": tpot}, + }, + } + + slow = {"benchmarks": [point(4, 100.0, 50.0, 10.0), point(8, 200.0, 80.0, 20.0)]} + fast = {"benchmarks": [point(4, 300.0, 400.0, 30.0), point(8, 600.0, 450.0, 40.0)]} + trials_dir = _write_trials(tmp_path, [slow, fast]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + best = summary["best"] + assert best is not None + assert best["concurrency"] == 8 + assert best["ttft_ms_range"] == [400.0, 450.0] + assert best["per_user_tok_s_range"] == [25.0, 1000 / 30.0] + + def test_a_partial_sweep_ranges_over_its_usable_points(self, tmp_path): + usable = { + "workload": {"concurrency": 4}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": 100.0, + "ttft_ms": {"p50": 200.0}, + "tpot_ms": {"mean": 20.0}, + }, + } + broken = {"workload": {"concurrency": 8}, "metrics": {"duration_seconds": 0}} + trials_dir = _write_trials(tmp_path, [{"benchmarks": [usable, broken]}]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + best = summary["best"] + assert best is not None + assert best["ttft_ms_range"] == [200.0, 200.0] + assert best["per_user_tok_s_range"] == [50.0, 50.0] + + def test_a_pre_sweep_record_has_a_single_point_range(self, tmp_path): + record = { + "benchmark": { + "workload": {"concurrency": 8}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": 100.0, + "ttft_ms": {"p50": 240.0}, + "tpot_ms": {"mean": 25.0}, + }, + } + } + trials_dir = _write_trials(tmp_path, [record]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + best = summary["best"] + assert best is not None + assert best["ttft_ms_range"] == [240.0, 240.0] + + def test_judges_a_sweep_that_breached_everywhere_by_its_closest_point(self, tmp_path): + record = { + "benchmarks": [ + { + "workload": {"concurrency": 4}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": 800.0, + "ttft_ms": {"p50": 700.0}, + }, + }, + { + "workload": {"concurrency": 8}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": 900.0, + "ttft_ms": {"p50": 900.0}, + }, + }, + ] + } + trials_dir = _write_trials(tmp_path, [record]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + best = summary["best"] + assert best is not None + assert best["concurrency"] == 4 + def test_a_torn_result_copy_reads_as_still_in_flight(self, tmp_path): trials_dir = _write_trials(tmp_path, []) (trials_dir / "1").mkdir(parents=True) diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index c6203023f4..c4c4dcf09e 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -863,7 +863,7 @@ def test_renders_defaults_for_the_optional_fields(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": 8, + "concurrency": [8], "input_tokens": 1024, "output_tokens": 1024, "shared_prefix_tokens": 0, @@ -897,7 +897,7 @@ def test_renders_custom_dataset_without_request_shape(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": 8, + "concurrency": [8], "dataset": "spec_bench", "baseline": True, "fleets": ["gpu-fleet"], @@ -927,7 +927,7 @@ def test_renders_configured_values(self): assert data["model"] == {"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"} assert data["min_context_length"] == 32768 assert data["trials_num"] == 10 - assert data["concurrency"] == 16 + assert data["concurrency"] == [16] assert data["fleets"] == ["a", "b"] diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index e5e963388b..749b971b5c 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -32,6 +32,7 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): ttft.mean = 8148.3 ttft.p50 = 8151.4 ttft.p99 = 8334.2 + preset.benchmarks = [preset.benchmark] output = output_module.format_preset_benchmark(preset, verbose=True) @@ -42,6 +43,28 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): assert "ttft=8.15s" in output assert "e+03" not in output + def test_shows_a_sweep_as_ranges_and_names_the_selected_point(self): + preset = get_preset() + points = [] + for concurrency, tpot, ttft, tokens in ((4, 30.0, 425.0, 1024), (16, 52.0, 861.0, 4096)): + point = preset.benchmark.model_copy(deep=True) + point.workload.concurrency = concurrency + point.metrics.tpot_ms.mean = tpot + point.metrics.ttft_ms.p50 = ttft + point.metrics.total_output_tokens = tokens + points.append(point) + preset.benchmarks = points + preset.benchmark = points[1] + + output = output_module.format_preset_benchmark(preset, verbose=True) + + assert output == "tps/user=19.2..33.3 c=16 tps=21.1..84.2 ttft=425ms..861ms ctx=32K" + + def test_shows_single_values_without_a_sweep(self): + output = output_module.format_preset_benchmark(get_preset()) + + assert output == "tps/user=133 ttft=108ms ctx=32K" + class TestFormatPresetObjective: def test_shows_the_requested_shared_prefix(self): @@ -140,7 +163,7 @@ def test_prints_submitted_column(self, monkeypatch): assert "2 months ago" in output.getvalue() -def _session_row(session: dict) -> dict: +def _session_row(session: dict, *, verbose: bool = False) -> dict: table = Table(box=None) for column in ( "BASE", @@ -154,7 +177,7 @@ def _session_row(session: dict) -> dict: "SUBMITTED", ): table.add_column(column) - _add_session(table, session) + _add_session(table, session, verbose=verbose) return { column.header: "".join(str(cell) for cell in column._cells) for column in table.columns } @@ -171,7 +194,7 @@ def test_shows_progress_after_status_and_best_benchmark(self): "count": 2, "best": { "tok_s": 2339.0, - "tpot_ms": 3.42, + "per_user_tok_s_range": [292.0, 292.0], "concurrency": 8, "gpu": "A40:48GB:1", }, @@ -202,6 +225,84 @@ def test_shows_the_shared_prefix_when_the_workload_has_one(self): assert row["CONSTRAINTS"] == ("[secondary]io=8K/1K prefix=90% c=162[/]") + def test_shows_a_sweep_and_the_point_the_best_trial_is_judged_by(self): + row = _session_row( + { + "id": "c7e18d52", + "status": "running", + "constraints": {"dataset": "spec_bench", "concurrency": [2, 4, 8, 16]}, + "trials": { + "count": 1, + "best": { + "tok_s": 2339.0, + "per_user_tok_s_range": [292.0, 292.0], + "concurrency": 8, + "gpu": None, + }, + }, + } + ) + + assert row["CONSTRAINTS"] == "[secondary]data=spec_bench c=2..16[/]" + assert row["BENCHMARK"].startswith("tps/user=292 c=8") + + def test_shows_a_sweep_session_as_ranges(self): + row = _session_row( + { + "id": "6ac0bec5", + "status": "interrupted", + "constraints": {"concurrency": [4, 8, 16]}, + "trials": { + "count": 1, + "best": { + "tok_s": 380.0, + "tok_s_range": [132.0, 380.0], + "per_user_tok_s_range": [24.3, 33.3], + "ttft_ms_range": [425.0, 861.0], + "context_length": 98304, + "concurrency": 16, + }, + }, + }, + verbose=True, + ) + + assert row["BENCHMARK"] == ( + "tps/user=24.3..33.3 c=16 tps=132..380 ttft=425ms..861ms ctx=96K" + ) + + def test_a_failed_sweep_keeps_its_ranges_and_its_mark(self): + row = _session_row( + { + "id": "6ac0bec5", + "status": "failed", + "constraints": {"concurrency": [4, 8]}, + "trials": { + "count": 1, + "best": None, + "best_failed": { + "tok_s": 200.0, + "per_user_tok_s_range": [25.0, 33.3], + "ttft_ms_range": [900.0, 1100.0], + "concurrency": 4, + }, + }, + } + ) + + assert row["BENCHMARK"] == "[secondary]*tps/user=25..33.3 c=4 ttft=900ms..1.1s[/]" + + def test_shows_a_sweep_that_does_not_double_as_a_list(self): + row = _session_row( + { + "id": "c7e18d52", + "status": "running", + "constraints": {"dataset": "spec_bench", "concurrency": [2, 3]}, + } + ) + + assert row["CONSTRAINTS"] == "[secondary]data=spec_bench c=2,3[/]" + def test_shows_the_dataset_for_a_session_with_a_custom_dataset(self): row = _session_row( { @@ -511,8 +612,8 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): "best": None, "best_failed": { "tok_s": 100.0, - "tpot_ms": 34.4, - "ttft_ms": 4300.0, + "per_user_tok_s_range": [29.1, 29.1], + "ttft_ms_range": [4300.0, 4300.0], "context_length": 1048576, "concurrency": 4, "gpu": "MI300X:192GB:1", @@ -527,6 +628,50 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): # styling is absent or invisible to the reader. assert "*tps/user" in row["BENCHMARK"] + def test_a_sweep_trial_record_reaches_the_row_as_ranges(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + # The summary writes the keys the row reads; only both together prove it. + def point(concurrency, tokens, ttft, tpot): + return { + "workload": {"concurrency": concurrency}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": tokens, + "ttft_ms": {"p50": ttft}, + "tpot_ms": {"mean": tpot}, + }, + } + + trials_dir = _write_trials( + tmp_path, + [ + { + "context_length": 98304, + "benchmarks": [ + point(4, 132.0, 425.0, 30.0), + point(8, 237.0, 691.0, 34.0), + point(16, 380.0, 861.0, 41.2), + ], + } + ], + ) + + trials = _summarize_session_trials(trials_dir, max_ttft=2000) + row = _session_row( + { + "id": "6ac0bec5", + "status": "interrupted", + "constraints": {"concurrency": [4, 8, 16], "max_ttft": 2000}, + "trials": trials, + }, + verbose=True, + ) + + assert row["BENCHMARK"] == ( + "tps/user=24.3..33.3 c=16 tps=132..380 ttft=425ms..861ms ctx=96K" + ) + class TestFailedTrialSpark: def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): diff --git a/src/tests/_internal/cli/services/presets/test_report_schema.py b/src/tests/_internal/cli/services/presets/test_report_schema.py index 9298e06d2a..0e9eb437e7 100644 --- a/src/tests/_internal/cli/services/presets/test_report_schema.py +++ b/src/tests/_internal/cli/services/presets/test_report_schema.py @@ -19,7 +19,7 @@ "base": None, "model": None, "context_length": None, - "benchmark": None, + "benchmarks": None, "failure_summary": None, } # Keywords the OpenAI strict structured-output mode rejects. @@ -50,6 +50,15 @@ def _walk(node): yield from _walk(item) +class TestGetReportJsonSchema: + def test_offers_only_the_sweep_shape(self): + # The legacy `benchmark` is accepted at parse only; agents are never offered it. + properties = get_report_json_schema()["properties"] + + assert "benchmarks" in properties + assert "benchmark" not in properties + + class TestGetStrictReportJsonSchema: def test_every_object_requires_all_of_its_properties(self): objects = [ @@ -106,31 +115,33 @@ def test_keeps_the_agent_facing_shape(self): "base": "Qwen/Qwen2.5-0.5B-Instruct", "model": "Qwen/Qwen2.5-0.5B-Instruct", "context_length": 2048, - "benchmark": { - "tool": "vllm.bench serve", - "tool_version": "0.27.0", - "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", - "workload": { - "api": "chat_completions", - "dataset": None, - "num_requests": 32, - "input_tokens": 128, - "output_tokens": 64, - "concurrency": 1, - "shared_prefix_tokens": 0, - }, - "metrics": { - "successful_requests": 32, - "failed_requests": 0, - "duration_seconds": 40.5, - "total_input_tokens": 4096, - "total_output_tokens": 2048, - "output_tok_per_s": 50.6, - "per_user_tok_per_s": 50.6, - "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, - "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, - }, - }, + "benchmarks": [ + { + "tool": "vllm.bench serve", + "tool_version": "0.27.0", + "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", + "workload": { + "api": "chat_completions", + "dataset": None, + "num_requests": 32, + "input_tokens": 128, + "output_tokens": 64, + "concurrency": 1, + "shared_prefix_tokens": 0, + }, + "metrics": { + "successful_requests": 32, + "failed_requests": 0, + "duration_seconds": 40.5, + "total_input_tokens": 4096, + "total_output_tokens": 2048, + "output_tok_per_s": 50.6, + "per_user_tok_per_s": 50.6, + "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, + "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, + }, + } + ], }, PresetAgentSuccess, ), @@ -146,5 +157,5 @@ def test_reports_codex_produced_against_the_schema_parse(self, data, expected): assert isinstance(report, expected) if isinstance(report, PresetAgentSuccess): assert isinstance(report.service_yaml, ServiceConfiguration) - assert report.benchmark.workload.dataset is None - assert report.benchmark.workload.shared_prefix_tokens == 0 + assert report.benchmarks[0].workload.dataset is None + assert report.benchmarks[0].workload.shared_prefix_tokens == 0 diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index fb774a68ff..0dbcc66929 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -383,7 +383,7 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert preset.configuration.input_tokens == 10000 assert preset.configuration.output_tokens == 1500 assert preset.configuration.shared_prefix_tokens == 0 - assert preset.configuration.concurrency == 4 + assert preset.configuration.concurrency == [4] # `priority` is not an excluded field, so the regrouping keeps it. assert preset.service.priority == 0 assert [group.name for group in preset.verified_on] == [ @@ -392,6 +392,8 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert get_or_error(preset.verified_on[0].replicas[0].gpu).name == ["MI300X"] # The old format never recorded a dataset name for a synthetic workload. assert preset.benchmark.workload.dataset is None + # One concurrency, so the one benchmark is the whole sweep. + assert preset.benchmarks == [preset.benchmark] assert store.list() == [preset] def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index c7a4d9ea80..97e2ab1321 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -4,7 +4,11 @@ import pytest from pydantic import ValidationError -from dstack._internal.cli.models.preset_agent import AnyPresetAgentResult, PresetAgentSuccess +from dstack._internal.cli.models.preset_agent import ( + AnyPresetAgentResult, + PresetAgentFailure, + PresetAgentSuccess, +) from dstack._internal.cli.services.presets.agent import ( PresetAgentProcessOutput, ) @@ -61,6 +65,7 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): base_model="Qwen/Qwen3.5-27B", repo="community/Qwen3.5-27B-GPTQ-Int4", context_length=32768, + benchmarks=[get_preset_benchmark()], benchmark=get_preset_benchmark(), configuration=configuration, best_trial=1, @@ -76,12 +81,12 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): assert preset.configuration.spot_policy is None assert "secret" not in preset.model_dump_json() - def test_successful_report_requires_benchmark(self): + def test_successful_report_requires_benchmarks(self): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() - data.pop("benchmark") + data.pop("benchmarks") - with pytest.raises(ValidationError, match="benchmark"): + with pytest.raises(ValidationError, match="benchmarks"): validate_extra_ignore(AnyPresetAgentResult, data) def test_builds_portable_self_contained_preset(self, tmp_path): @@ -222,11 +227,11 @@ def test_rejects_a_benchmark_without_the_requested_shared_prefix(self, tmp_path) created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): + def test_rejects_benchmarks_at_other_concurrencies(self, tmp_path): run = get_running_service_run() with pytest.raises( - CLIError, match="concurrency of 8 does not match the requested concurrency of 4" + CLIError, match=r"concurrencies \[8\] do not match the requested \[4\]" ): build_verified_preset( run=run, @@ -239,6 +244,44 @@ def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) + def test_rejects_a_partial_sweep(self, tmp_path): + run = get_running_service_run() + + with pytest.raises( + CLIError, match=r"concurrencies \[2, 4\] do not match the requested \[2, 4, 8\]" + ): + build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(concurrency="2..8"), + report=_sweep_report(run, [2, 4]), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_stores_every_point_and_selects_the_fastest_within_max_ttft(self, tmp_path): + run = get_running_service_run() + + preset = build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(concurrency="2..8", max_ttft=500), + # Reported out of order; throughput rises with concurrency, but the top + # point breaches the TTFT cap. + report=_sweep_report( + run, [8, 2, 4], duration_seconds={2: 8.0, 4: 4.0, 8: 2.0}, ttft_p50={8: 900.0} + ), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert [b.workload.concurrency for b in preset.benchmarks] == [2, 4, 8] + assert preset.benchmark.workload.concurrency == 4 + def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): run = get_running_service_run() report = _dataset_report(run, dataset="spec_bench") @@ -346,22 +389,22 @@ def test_redacts_known_secret_in_benchmark_command_instead_of_failing(self, tmp_ run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["benchmarks"][0]["command"] = ( "python bench.py --header 'Authorization: Bearer sk-live-0123456789abcdef'" ) report = self._load(tmp_path, data, redacted_values=("sk-live-0123456789abcdef",)) assert isinstance(report, PresetAgentSuccess) - assert report.benchmark is not None - assert report.benchmark.command.endswith("Bearer [redacted]'") - assert "sk-live" not in report.benchmark.command + assert report.benchmarks + assert report.benchmarks[0].command.endswith("Bearer [redacted]'") + assert "sk-live" not in report.benchmarks[0].command def test_still_rejects_unknown_bearer_token(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["benchmarks"][0]["command"] = ( "curl -H 'Authorization: Bearer sk-unknown-9876543210fedcba'" ) @@ -374,7 +417,7 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["benchmarks"][0]["command"] = ( "./benchenv/bin/python bench_service.py --base $DSTACK_SERVER_URL/x" " (auth via DSTACK_TOKEN bearer header from env)" ) @@ -382,11 +425,46 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): report = self._load(tmp_path, data, redacted_values=()) assert isinstance(report, PresetAgentSuccess) - assert report.benchmark is not None - assert "bearer header" in report.benchmark.command + assert report.benchmarks + assert "bearer header" in report.benchmarks[0].command + + def test_reads_a_pre_sweep_success_as_a_one_point_sweep(self, tmp_path): + # A session detached under a pre-sweep CLI finishes with that CLI's schema. + run = get_running_service_run() + data = get_successful_preset_report(run).model_dump() + data["run_id"] = str(data["run_id"]) + data["benchmark"] = data.pop("benchmarks")[0] + + report = self._load(tmp_path, data, redacted_values=()) + + assert isinstance(report, PresetAgentSuccess) + assert [b.workload.concurrency for b in report.benchmarks] == [1] + + def test_reads_a_sweep_success_unchanged(self, tmp_path): + run = get_running_service_run() + data = get_successful_preset_report(run).model_dump() + data["run_id"] = str(data["run_id"]) + second = {**data["benchmarks"][0], "workload": {**data["benchmarks"][0]["workload"]}} + second["workload"]["concurrency"] = 2 + data["benchmarks"].append(second) + + report = self._load(tmp_path, data, redacted_values=()) + + assert isinstance(report, PresetAgentSuccess) + assert [b.workload.concurrency for b in report.benchmarks] == [1, 2] + + def test_leaves_a_failure_report_alone(self, tmp_path): + report = self._load( + tmp_path, + {"success": False, "failure_summary": "no configuration met max_ttft"}, + redacted_values=(), + ) + + assert isinstance(report, PresetAgentFailure) + assert report.failure_summary == "no configuration met max_ttft" -def _shared_prefix_configuration() -> PresetConfiguration: +def _shared_prefix_configuration(**overrides: Any) -> PresetConfiguration: """The configuration from dstackai/dstack#4198: a shared prefix and no dataset.""" return PresetConfiguration.model_validate( { @@ -399,6 +477,7 @@ def _shared_prefix_configuration() -> PresetConfiguration: "input_tokens": 131072, "output_tokens": 512, "shared_prefix_tokens": 130048, + **overrides, } ) @@ -407,6 +486,29 @@ def _shared_prefix_report(run: Run, **workload: Any) -> PresetAgentSuccess: return _report_with_workload(run, {**SHARED_PREFIX_WORKLOAD, **workload}) +def _sweep_report( + run: Run, + concurrencies: list[int], + *, + duration_seconds: dict[int, float] | None = None, + ttft_p50: dict[int, float] | None = None, +) -> PresetAgentSuccess: + """One shared-prefix benchmark per concurrency, in the given order. The + metrics are the fixture's unless overridden per concurrency.""" + benchmarks = [] + for concurrency in concurrencies: + benchmark = get_preset_benchmark() + benchmark.workload = PresetWorkload.model_validate( + {**SHARED_PREFIX_WORKLOAD, "concurrency": concurrency} + ) + if duration_seconds and concurrency in duration_seconds: + benchmark.metrics.duration_seconds = duration_seconds[concurrency] + if ttft_p50 and concurrency in ttft_p50: + benchmark.metrics.ttft_ms.p50 = ttft_p50[concurrency] + benchmarks.append(benchmark) + return get_successful_preset_report(run).model_copy(update={"benchmarks": benchmarks}) + + def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: # A dataset defines the request shape, so the workload records what it measured. return _report_with_workload( @@ -425,4 +527,4 @@ def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: def _report_with_workload(run: Run, workload: dict[str, Any]) -> PresetAgentSuccess: benchmark = get_preset_benchmark() benchmark.workload = PresetWorkload.model_validate(workload) - return get_successful_preset_report(run).model_copy(update={"benchmark": benchmark}) + return get_successful_preset_report(run).model_copy(update={"benchmarks": [benchmark]}) diff --git a/src/tests/_internal/cli/services/presets/test_workspace.py b/src/tests/_internal/cli/services/presets/test_workspace.py index a10832688e..3c60ab227a 100644 --- a/src/tests/_internal/cli/services/presets/test_workspace.py +++ b/src/tests/_internal/cli/services/presets/test_workspace.py @@ -1,3 +1,5 @@ +import json + import pytest from dstack._internal.cli.services.presets.session import PresetSession @@ -81,3 +83,60 @@ def test_a_session_without_records_warns(self, tmp_path, capsys): assert "empty000 has no records" in capsys.readouterr().err assert not (workspace.path / "previous" / "empty000").exists() + + def test_upgrades_pre_sweep_records_to_the_sweep_format(self, tmp_path): + session = _previous_session(tmp_path) + root = session.path + benchmark = {"workload": {"concurrency": 8}, "metrics": {}} + (root / "constraints.json").write_text(json.dumps({"max_ttft": 2000, "concurrency": 8})) + (root / "trials" / "1" / "trial.json").write_text( + json.dumps({"context_length": 8192, "benchmark": benchmark, "learned": "x"}) + ) + (root / "trials" / "2").mkdir() + # A trial that never produced a benchmark. + (root / "trials" / "2" / "trial.json").write_text( + json.dumps({"benchmark": None, "learned": "y", "failed": True}) + ) + (root / "final_report.json").write_text( + json.dumps({"success": True, "benchmark": benchmark}) + ) + workspace = _workspace(tmp_path) + + install_previous_records(workspace, [session]) + + target = workspace.path / "previous" / "8d3b01aa" + assert json.loads((target / "constraints.json").read_text()) == { + "max_ttft": 2000, + "concurrency": [8], + } + trial = json.loads((target / "trials" / "1" / "trial.json").read_text()) + # Same position as the field it replaces. + assert list(trial) == ["context_length", "benchmarks", "learned"] + assert trial["benchmarks"] == [benchmark] + failed = json.loads((target / "trials" / "2" / "trial.json").read_text()) + assert failed["benchmarks"] is None + report = json.loads((target / "final_report.json").read_text()) + assert report == {"success": True, "benchmarks": [benchmark]} + # The session's own records are not rewritten. + assert json.loads((root / "constraints.json").read_text())["concurrency"] == 8 + + def test_copies_sweep_records_unchanged(self, tmp_path): + session = _previous_session(tmp_path) + text = '{"benchmarks": [{"workload": {"concurrency": 4}}], "learned": "x"}' + (session.path / "trials" / "1" / "trial.json").write_text(text) + workspace = _workspace(tmp_path) + + install_previous_records(workspace, [session]) + + copied = workspace.path / "previous" / "8d3b01aa" / "trials" / "1" / "trial.json" + assert copied.read_text() == text + + def test_copies_an_unreadable_record_as_written(self, tmp_path): + session = _previous_session(tmp_path) + (session.path / "trials" / "1" / "trial.json").write_text('{"benchmark": {"met') + workspace = _workspace(tmp_path) + + install_previous_records(workspace, [session]) + + copied = workspace.path / "previous" / "8d3b01aa" / "trials" / "1" / "trial.json" + assert copied.read_text() == '{"benchmark": {"met' diff --git a/src/tests/_internal/core/models/test_configurations.py b/src/tests/_internal/core/models/test_configurations.py index a77656c43d..ec9f35a65e 100644 --- a/src/tests/_internal/core/models/test_configurations.py +++ b/src/tests/_internal/core/models/test_configurations.py @@ -1344,6 +1344,49 @@ def test_defaults_to_a_synthetic_workload(self): assert configuration.dataset is None +class TestPresetConfigurationConcurrency: + @pytest.mark.parametrize( + ("value", "expected"), + [ + pytest.param(10, [10], id="int"), + pytest.param("2,3", [2, 3], id="comma"), + pytest.param([3, 2, 2], [2, 3], id="list-sorted-deduped"), + pytest.param("2..16", [2, 4, 8, 16], id="range"), + pytest.param("8..8", [8], id="range-single"), + pytest.param(" 2 .. 8 ", [2, 4, 8], id="range-spaces"), + ], + ) + def test_normalizes_to_list(self, value, expected): + conf = PresetConfiguration.model_validate( + {"type": "preset", "base": "m", "concurrency": value} + ) + + assert conf.concurrency == expected + + @pytest.mark.parametrize( + ("value", "error"), + [ + pytest.param("3..16", "not reachable by doubling from 3", id="range-off-chain"), + pytest.param("..16", "needs both ends", id="range-no-start"), + pytest.param("2..", "needs both ends", id="range-no-end"), + pytest.param("16..2", "up to its end", id="range-descending"), + pytest.param([], "at least 1", id="empty-list"), + pytest.param(0, "greater than 0", id="zero"), + pytest.param("2,x", "valid integer", id="comma-garbage"), + ], + ) + def test_rejects(self, value, error): + with pytest.raises(ValidationError, match=error): + PresetConfiguration.model_validate( + {"type": "preset", "base": "m", "concurrency": value} + ) + + def test_stays_unset_when_omitted(self): + conf = PresetConfiguration.model_validate({"type": "preset", "base": "m"}) + + assert conf.concurrency is None + + class TestPresetConfigurationSchema: def test_schema_does_not_require_model(self): # `model` is filled from the `base`/`repo` shorthand by a before-validator, diff --git a/src/tests/_internal/core/models/test_presets.py b/src/tests/_internal/core/models/test_presets.py index e12a881663..bd1f8133d3 100644 --- a/src/tests/_internal/core/models/test_presets.py +++ b/src/tests/_internal/core/models/test_presets.py @@ -5,6 +5,7 @@ from dstack._internal.core.models.presets import ( PresetBenchmark, + select_benchmark, validate_preset_file_path, validate_preset_file_paths, ) @@ -152,3 +153,32 @@ def test_keeps_a_dataset_named_random_without_a_prefix(self): benchmark = PresetBenchmark.model_validate(data) assert benchmark.workload.dataset == "random" + + +def _benchmark(*, concurrency: int, tok_s: float, ttft_p50: float) -> PresetBenchmark: + data = get_benchmark_data(get_workload_data(concurrency=concurrency)) + # The fixture's 2048 output tokens over `duration` give `tok_s`. + data["metrics"]["duration_seconds"] = 2048 / tok_s + data["metrics"]["ttft_ms"]["p50"] = ttft_p50 + return PresetBenchmark.model_validate(data) + + +class TestSelectBenchmark: + def test_picks_the_fastest_point_within_max_ttft(self): + over = _benchmark(concurrency=16, tok_s=600, ttft_p50=900) + ok = _benchmark(concurrency=8, tok_s=500, ttft_p50=300) + + assert select_benchmark([ok, over], max_ttft=500) is ok + + def test_falls_back_to_the_closest_ttft_when_none_passes(self): + # Everything breached: the record is how near it got, not how fast it ran. + closer = _benchmark(concurrency=8, tok_s=500, ttft_p50=900) + faster = _benchmark(concurrency=16, tok_s=600, ttft_p50=950) + + assert select_benchmark([closer, faster], max_ttft=500) is closer + + def test_breaks_a_ttft_tie_by_throughput(self): + slow = _benchmark(concurrency=8, tok_s=500, ttft_p50=900) + fast = _benchmark(concurrency=16, tok_s=600, ttft_p50=900) + + assert select_benchmark([slow, fast], max_ttft=500) is fast From 8e1efac477409a29761c5ddef091e732d46fb3ba Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Thu, 5 Nov 2026 15:44:51 +0545 Subject: [PATCH 02/12] Minor Update --- .../_internal/cli/models/preset_agent.py | 6 ++---- .../_internal/cli/services/presets/output.py | 8 +++---- .../presets/resources/system_prompt.md | 3 +-- .../_internal/cli/services/presets/session.py | 21 ++++++++++++------- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 918e622737..c6574aa7bb 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -59,10 +59,8 @@ class PresetAgentSuccess(CoreModel): @model_validator(mode="before") @classmethod def wrap_legacy_single_benchmark(cls, data: Any) -> Any: - # An agent process started by a pre-sweep CLI reports one `benchmark`, and - # a session detached before the upgrade is attached and verified by the - # new CLI. The one point is its whole sweep; the sweep check in `verify` - # still requires it to be the requested concurrency. + # Older agents report a single `benchmark`. Wrap it in a list so + # the new CLI can read reports from sessions started before the upgrade. if isinstance(data, dict) and "benchmark" in data and "benchmarks" not in data: data = dict(data) data["benchmarks"] = [data.pop("benchmark")] diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index f5cf9fde07..3e75d07e50 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -214,7 +214,7 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False per_user = best.get("per_user_tok_s_range") if per_user: parts.append(f"tps/user={_format_range(per_user, _format_number)}") - # Which point of a sweep the numbers are from; redundant for one value. + # Show the selected concurrency when the sweep has multiple values. if isinstance(sweep, list) and len(sweep) > 1 and best.get("concurrency"): parts.append(f"c={best['concurrency']}") if verbose: @@ -349,8 +349,7 @@ def _breaches_constraints(preset: AnyStoredPreset) -> bool: def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) -> str: - # A sweep shows each metric as its range across concurrencies, and `c=` names - # the selected point. + # Show benchmark ranges across the sweep and identify the selected concurrency. benchmarks = preset.benchmarks parts = [ "tps/user=" @@ -379,8 +378,7 @@ def _format_range(values: list[float], fmt: Callable[[float], str]) -> str: def _format_concurrency(value: Any) -> str: - """A sweep in the shape the user wrote it: `8`, `2,3`, or `2..16` when every - step doubles. A pre-sweep record carries a bare int.""" + """Format concurrency values for display, for example: 8, 2,3, or 2..16.""" if not isinstance(value, list): return str(value) if len(value) > 1 and all(b == a * 2 for a, b in zip(value, value[1:])): diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 79605eeec3..80a282c185 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -333,8 +333,7 @@ During trials, run benchmarks via SSH inside the task, directly against the serving engine: run one benchmark per value of `concurrency`, using `dataset``input_tokens`, `output_tokens`, and `shared_prefix_tokens` from `constraints.json`, and measure all trials the same -way so that their results are comparable with each other. Use a different -seed for each point of the sweep. +way so that their results are comparable with each other. In case the task is using a router (see `## Cluster Placement`), run benchmarks via SSH inside the router node, directly against the router engine, never against a diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index dc09cc2b5d..e56eeadd28 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -642,11 +642,16 @@ def _summarize_session_trials( def _select_trial_point( record: dict[str, Any], max_ttft: Optional[float] ) -> Optional[tuple[float, dict[str, Any], dict[str, Any], list[dict[str, Any]]]]: - """The sweep point a trial is judged by, as `select_benchmark` picks it for a - preset: the highest throughput within `max_ttft`, else the closest TTFT. - A record from before the sweep holds a single `benchmark`. Returns - `(tok_s, metrics, workload, sweep)`, where `sweep` is the metrics of every - point with usable numbers, or `None` when there is none.""" + """Select the benchmark point used to judge a trial. + + Choose the highest-throughput point whose p50 TTFT meets `max_ttft`. + If no point meets the limit, choose the point with the lowest p50 TTFT. + Older trial records may contain a single `benchmark` instead of `benchmarks`. + + Return `(tok_s, metrics, workload, sweep)`, where `metrics` and `workload` + describe the selected point and `sweep` contains metrics for all usable + points. Return `None` if no point has usable throughput data. + """ benchmarks = record.get("benchmarks") if not isinstance(benchmarks, list): benchmarks = [record.get("benchmark")] @@ -698,8 +703,10 @@ def _trial_entry( "tok_s": tok_s, "tpot_ms": tpot if isinstance(tpot, (int, float)) and tpot > 0 else None, "ttft_ms": ttft if isinstance(ttft, (int, float)) else None, - # The trial's spread across its sweep, as `[low, high]`, None when no - # point reports the value. One point gives `low == high`. + # Store the lowest and highest throughput and TTFT values across the trial's + # concurrency points. For example, throughput values 132, 237, and 380 + # tokens/sec become [132, 380]. One value becomes [value, value]; no values + # becomes None. "tok_s_range": _range([m["total_output_tokens"] / m["duration_seconds"] for m in sweep]), "per_user_tok_s_range": _range( [1000 / v for v in (_latency(m, "tpot_ms", "mean") for m in sweep) if v] From 7533695ee44387f0551e91fec41a67e44eec26fb Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Sun, 8 Nov 2026 08:34:22 +0545 Subject: [PATCH 03/12] Update system_prompt.md Update system_prompt.md Update system_prompt.md Update system_prompt.md Update system_prompt.md Update system_prompt.md --- .../presets/resources/system_prompt.md | 116 ++++++++++-------- 1 file changed, 68 insertions(+), 48 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 80a282c185..0c5276a0b3 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -5,21 +5,9 @@ hardware, through sequential experimental trials. ## Performance -Performance is the highest `output_tok_per_s` a configuration reaches across -the `concurrency` sweep from `constraints.json` (see `## Benchmark`), counting -only sweep points that meet `max_ttft`, the maximum p50 time to first token, in -milliseconds (see `# Constraints`). Points that breach `max_ttft` are ignored. -For example, with `max_ttft` of 2000: - -| `concurrency` | `output_tok_per_s` | `ttft_ms.p50` | -|---|---|---| -| 4 | 130 | 400 | -| 8 | 240 | 1500 | -| 16 | 380 | 2400 | - -The point at 16 breaches `max_ttft`, so it is ignored. Performance is 240, the -highest throughput among the remaining points. Benchmark tools report several -throughput numbers; performance is the output one. +Performance is `output_tok_per_s`, the output token throughput of a benchmark +run at the `concurrency` from `constraints.json` (see `## Benchmark`). Benchmark +tools report several throughput numbers; performance is the output one. Significantly better performance goes beyond what conventional configurations and published benchmarks reach; they are not the ceiling. Estimate the @@ -47,9 +35,10 @@ Field semantics: - `max_ttft`: the maximum p50 time to first token, in milliseconds, that any benchmark in this session may report. - `trials_num`: the number of trials in this session. -- `concurrency`: the numbers of simultaneous requests every benchmark in - this session is run at, one benchmark per value. The list is fixed so - that benchmark results are comparable across trials. +- `concurrency`: a list of simultaneous request counts used for every + benchmark in this session. For example, `[2, 4, 8]` means running the + benchmark once each at 2, 4, and 8 simultaneous requests. The list is + fixed so that benchmark results are comparable across trials. - `dataset`: the benchmark dataset for every benchmark in this session; see `## Benchmark`. @@ -257,7 +246,7 @@ benchmark results (see `## Benchmark` for the structure) to marks trial `` completed. The benchmark may be skipped in one case only: you failed to make the configuration run at all — a failed trial. `trials//task.dstack.yml` may be skipped in -one case only: you failed to get benchmark results at all. A trial is +one case only: you failed to get benchmark results at any concurrency. A trial is also failed when its benchmark does not meet the constraints (see `# Constraints`). When a trial that changed several things fails, be mindful of which specific change was the root cause. @@ -268,7 +257,7 @@ mindful of which specific change was the root cause. others: ``` -{"resources": {...}, "context_length": ..., "benchmarks": [{...}, ...], "learned": ..., "failed": ...} +{"resources": {...}, "context_length": [{...}, ...], "benchmark": [{...}, ...], "learned": [{...}, ...], "failed": [{...}, ...]} ``` 2. In case the task is using node groups (a router group plus worker @@ -276,7 +265,7 @@ mindful of which specific change was the root cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": ..., "benchmarks": [{...}, ...], "learned": ..., "failed": ...} +{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": [{...}, ...], "benchmark": [{...}, ...], "learned": [{...}, ...], "failed": [{...}, ...]} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -304,17 +293,50 @@ mindful of which specific change was the root cause. [{"cpu": "192", "memory": "2048GB", "disk": "1000GB", "gpu": {"name": "H200", "memory": "141GB", "count": 8}}] ] ``` -- `context_length`: the largest context the trial's configuration handles, - found as described in `## Benchmark`; `null` only when the benchmark couldn't - be done at all. -- `benchmarks`: one trial benchmark per value of `concurrency`, in that - order (see `## Benchmark` for the structure); `null` only when no benchmark - could be done at all. -- `learned`: the major things this trial taught you that you did not know - before it ran. Required for every trial, including a failed one. -- `failed`: `true` if the configuration never ran or the benchmark broke a - constraint such as `min_context_length`, or no sweep point met `max_ttft`; - absent otherwise. Breaching `max_ttft` at some points only is not a failure. +- `context_length`: a list of the largest context lengths the trial's configuration + handles at each concurrency, found as described in `## Benchmark`. Use `null` + for a concurrency when the benchmark could not be done at that concurrency. + + E.g. context lengths recorded for concurrency values 4 and 8: + + ``` + [ + {"concurrency": 4, "value": 32768}, + {"concurrency": 8, "value": 16384} + ] + ``` +- `benchmark`: a list of trial benchmark results (see `## Benchmark` for the + structure). Include a result for each concurrency at which the benchmark + completed. If the benchmark could not be done at a concurrency, omit its + result. +- `learned`: a list of the major things this trial taught you at each concurrency + that you did not know before the trial ran. Required for every concurrency in + every trial, including failed ones. + + E.g. observations recorded for concurrency values 4 and 8: + + ``` + [ + {"concurrency": 4, "value": "The configuration handled the workload."}, + {"concurrency": 8, "value": "Memory pressure reduced the supported context."} + ] + ``` +- `failed`: a list indicating whether the trial failed at each concurrency. Set + the value to `true` if the configuration did not run or the benchmark broke a + constraint such as `min_context_length` or `max_ttft` at that concurrency; + otherwise set it to `false`. + + E.g. the result for concurrency 4 passed, while the result for concurrency 8 + failed: + + ``` + [ + {"concurrency": 4, "value": false}, + {"concurrency": 8, "value": true} + ] + ``` + + A trial fails if `failed` is `true` for any concurrency. You're expected to do exactly `trials_num` trials (see `# Constraints`). @@ -389,10 +411,9 @@ never part of the measured metrics. All verification and benchmark requests must succeed. -Record every benchmark using the following structure and field names, one -object per value of `concurrency` — trial benchmarks in -`trials//trial.json.benchmarks`, the final benchmarks as -`final_report.json.benchmarks` (values are illustrative): +Record every benchmark using the following structure and field names — +trial benchmarks in `trials//trial.json`, the final benchmark as +`final_report.json.benchmark` (values are illustrative): ```json { @@ -422,17 +443,17 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free -invocation. For the final benchmarks, run each with streaming responses, one +invocation. For the final benchmark, run each with streaming responses, one per value of `concurrency` from `constraints.json` with `workload.concurrency` set to that value, produce every field of the structure, and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and warmup requests. Never invent missing values. -After each benchmark, find the largest context the configuration handles by -sending real requests, and record it: for a trial, as the `context_length` -field of `trials//trial.json`; for the final benchmark, as -`final_report.json.context_length`. Stopping at the required minimum is not -enough. +After each benchmark, find the largest context the configuration handles at +each concurrency by sending real requests, and record the results: for a trial, in the +`context_length` list of `trials//trial.json`; for the final service, in +the `final_report.json.context_length` list. Stopping at the required minimum +is not enough. @@ -609,12 +630,12 @@ variable listed under `env` in `constraints.json`. Do not put secret values in `final_report.json`, or print them. Use env references in `final_report.json.service_yaml`; use environment variable names or redacted values in -`final_report.json.benchmarks[].command`. +`final_report.json.benchmark[].command`. # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmarks`, +`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmark`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed @@ -627,10 +648,9 @@ trial remained), include exactly: - `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `context_length`: the largest context verified for the final service, as - described in `## Benchmark` -- `benchmarks`: the final service benchmarks described in `## Benchmark`, - one per value of `concurrency` +- `context_length`: the largest context lengths verified for the final service, + as described in `## Benchmark` +- `benchmark`: the final service benchmark described in `## Benchmark` Set `final_report.json.base` as follows: From 5995cf67e0a981dd999964f04ff44ee902cb6512 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Tue, 10 Nov 2026 09:20:34 +0545 Subject: [PATCH 04/12] Update system_prompt.md Update system_prompt.md --- .../presets/resources/system_prompt.md | 147 ++++++++---------- 1 file changed, 67 insertions(+), 80 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 0c5276a0b3..5ef5a4ad7c 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -6,8 +6,8 @@ hardware, through sequential experimental trials. ## Performance Performance is `output_tok_per_s`, the output token throughput of a benchmark -run at the `concurrency` from `constraints.json` (see `## Benchmark`). Benchmark -tools report several throughput numbers; performance is the output one. +run at each `concurrency` value from `constraints.json` (see `## Benchmark`). +Benchmark tools report several throughput numbers; performance is the output one. Significantly better performance goes beyond what conventional configurations and published benchmarks reach; they are not the ceiling. Estimate the @@ -36,8 +36,8 @@ Field semantics: benchmark in this session may report. - `trials_num`: the number of trials in this session. - `concurrency`: a list of simultaneous request counts used for every - benchmark in this session. For example, `[2, 4, 8]` means running the - benchmark once each at 2, 4, and 8 simultaneous requests. The list is + benchmark in this session. For example, `[2, 4, 8]` means measuring + performance at each of 2, 4, and 8 simultaneous requests. The list is fixed so that benchmark results are comparable across trials. - `dataset`: the benchmark dataset for every benchmark in this session; see @@ -55,6 +55,15 @@ Field semantics: - `env`: the environment variable names available to runs; the values are already present in the environment and must never be printed or recorded. +Each trial is benchmarked at every value of `concurrency`, so a trial can have +one `output_tok_per_s` value for each concurrency. Before comparing trials, +decide on a method to compare their `output_tok_per_s` values across +concurrencies, and keep that method for the whole session. Report the chosen +method via `progress` (see `# Progress`). If `## Additional instructions` +says how to compare trials, use that method. + + During the trials and experimentation aimed at the best performance, you may pick the hardware (the best available within the allowed `dstack` fleets), the model variant (only if `model` has `base`), the serving framework, the @@ -243,13 +252,14 @@ Docker images and other dependencies.--> Once `trials//task.dstack.yml` is written, write the corresponding benchmark results (see `## Benchmark` for the structure) to `trials//trial.json`: the presence of `trials//trial.json` is what -marks trial `` completed. The benchmark -may be skipped in one case only: you failed to make the configuration run -at all — a failed trial. `trials//task.dstack.yml` may be skipped in -one case only: you failed to get benchmark results at any concurrency. A trial is -also failed when its benchmark does not meet the constraints (see -`# Constraints`). When a trial that changed several things fails, be -mindful of which specific change was the root cause. +marks trial `` completed. The benchmark at a concurrency may be skipped +only if you failed to make the configuration run at that concurrency. +`trials//task.dstack.yml` may be skipped only if the benchmark produced no +result at any concurrency value in `constraints.json` — a failed trial. A +trial is also failed when its benchmark at all concurrency values fails to +meet the constraints (see `# Constraints`). When a trial that changed +several things fails, be mindful of which specific change was the root +cause. `trials//trial.json` is a JSON object. @@ -257,7 +267,7 @@ mindful of which specific change was the root cause. others: ``` -{"resources": {...}, "context_length": [{...}, ...], "benchmark": [{...}, ...], "learned": [{...}, ...], "failed": [{...}, ...]} +{"resources": {...}, "results": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` 2. In case the task is using node groups (a router group plus worker @@ -265,7 +275,7 @@ mindful of which specific change was the root cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": [{...}, ...], "benchmark": [{...}, ...], "learned": [{...}, ...], "failed": [{...}, ...]} +{"groups": [[{...}], [{...}, {...}], [{...}]], "results": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -293,50 +303,22 @@ mindful of which specific change was the root cause. [{"cpu": "192", "memory": "2048GB", "disk": "1000GB", "gpu": {"name": "H200", "memory": "141GB", "count": 8}}] ] ``` -- `context_length`: a list of the largest context lengths the trial's configuration - handles at each concurrency, found as described in `## Benchmark`. Use `null` - for a concurrency when the benchmark could not be done at that concurrency. - - E.g. context lengths recorded for concurrency values 4 and 8: - - ``` - [ - {"concurrency": 4, "value": 32768}, - {"concurrency": 8, "value": 16384} - ] - ``` -- `benchmark`: a list of trial benchmark results (see `## Benchmark` for the - structure). Include a result for each concurrency at which the benchmark - completed. If the benchmark could not be done at a concurrency, omit its - result. -- `learned`: a list of the major things this trial taught you at each concurrency - that you did not know before the trial ran. Required for every concurrency in - every trial, including failed ones. - - E.g. observations recorded for concurrency values 4 and 8: - - ``` - [ - {"concurrency": 4, "value": "The configuration handled the workload."}, - {"concurrency": 8, "value": "Memory pressure reduced the supported context."} - ] - ``` -- `failed`: a list indicating whether the trial failed at each concurrency. Set - the value to `true` if the configuration did not run or the benchmark broke a - constraint such as `min_context_length` or `max_ttft` at that concurrency; - otherwise set it to `false`. - - E.g. the result for concurrency 4 passed, while the result for concurrency 8 - failed: - - ``` - [ - {"concurrency": 4, "value": false}, - {"concurrency": 8, "value": true} - ] - ``` - - A trial fails if `failed` is `true` for any concurrency. +- `results`: a list containing one entry for each value of `concurrency` + from `constraints.json`. Each entry contains: + + - `concurrency`: the concurrency value for this result. + - `context_length`: the largest context the trial's configuration handles + at this concurrency, found as described in `## Benchmark`; `null` only + when the benchmark couldn't be done at this concurrency. + - `benchmark`: the trial benchmark at this concurrency (see `## Benchmark` + for the structure); `null` only when the benchmark couldn't be done at this + concurrency. + - `learned`: the major things this trial taught you at this concurrency that + you did not know before it ran. Required for every `results` entry, + including a failed one. + - `failed`: `true` if the configuration did not run or the benchmark broke + a constraint such as `max_ttft` or `min_context_length` at this + concurrency; `false` otherwise. You're expected to do exactly `trials_num` trials (see `# Constraints`). @@ -352,9 +334,8 @@ no trials remain. In that case, log the failure to `final_report.json` (see ## Benchmark During trials, run benchmarks via SSH inside the task, directly against the -serving engine: run one benchmark per value of `concurrency`, using -`dataset``input_tokens`, `output_tokens`, and -`shared_prefix_tokens` from `constraints.json`, and measure all trials the same +serving engine: use `dataset` and `concurrency``concurrency`, `input_tokens`, `output_tokens`, and +`shared_prefix_tokens` from `constraints.json` and measure all trials the same way so that their results are comparable with each other. In case the task is using a router (see `## Cluster Placement`), run benchmarks @@ -411,9 +392,11 @@ never part of the measured metrics. All verification and benchmark requests must succeed. -Record every benchmark using the following structure and field names — -trial benchmarks in `trials//trial.json`, the final benchmark as -`final_report.json.benchmark` (values are illustrative): +Record every benchmark using the following structure and field names (values +are illustrative): for a trial, in `trials//trial.json` as +`results[concurrency == ].benchmark`; for the final service, as +`final_report.json.results[concurrency == ].benchmark`, where `` is +a concurrency value from `constraints.json`. ```json { @@ -443,17 +426,19 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free -invocation. For the final benchmark, run each with streaming responses, one -per value of `concurrency` from `constraints.json` with `workload.concurrency` -set to that value, produce every field of the structure, and calculate all metrics from the +invocation. For the final benchmark, use streaming responses. Set +`results[].benchmark.workload.concurrency` to the same value as +`results[].concurrency`. Produce every field of the structure, +and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and warmup requests. Never invent missing values. -After each benchmark, find the largest context the configuration handles at -each concurrency by sending real requests, and record the results: for a trial, in the -`context_length` list of `trials//trial.json`; for the final service, in -the `final_report.json.context_length` list. Stopping at the required minimum -is not enough. +After each benchmark, find the largest context the configuration handles by +sending real requests, and record it: for a trial, in +`trials//trial.json` as `results[concurrency == ].context_length`; +for the final service, as +`final_report.json.results[concurrency == ].context_length`, where `` +is a concurrency value. Stopping at the required minimum is not enough. @@ -627,16 +612,14 @@ print, copy, or summarize tokens, secrets, or environment variable values. Do not expose the value of `DSTACK_TOKEN` or the value of any environment variable listed under `env` in `constraints.json`. -Do not put secret values in `final_report.json`, or print -them. Use env -references in `final_report.json.service_yaml`; use environment variable names or redacted values in -`final_report.json.benchmark[].command`. +Do not put secret values in `final_report.json`, or print them. Use env +references in `final_report.json.service_yaml`; use environment variable +names or redacted values in `final_report.json.results[].benchmark.command`. # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmark`, -and `failure_summary`. +`service_yaml`, `trial`, `base`, `model`, `results`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed trial remained), include exactly: @@ -648,9 +631,13 @@ trial remained), include exactly: - `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `context_length`: the largest context lengths verified for the final service, - as described in `## Benchmark` -- `benchmark`: the final service benchmark described in `## Benchmark` +- `results`: the final service results, one entry for each value of `concurrency` + from `constraints.json`. Each entry contains: + - `concurrency`: the concurrency value for this result + - `context_length`: the largest context verified for the final service at + this concurrency, as described in `## Benchmark` + - `benchmark`: the final service benchmark at this concurrency, as + described in `## Benchmark` Set `final_report.json.base` as follows: From c9528229c32cbf8043fb842358dd3983cd0f8772 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Fri, 13 Nov 2026 12:37:09 +0545 Subject: [PATCH 05/12] Update system_prompt.md --- .../presets/resources/system_prompt.md | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 5ef5a4ad7c..d0dc937cae 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -55,15 +55,6 @@ Field semantics: - `env`: the environment variable names available to runs; the values are already present in the environment and must never be printed or recorded. -Each trial is benchmarked at every value of `concurrency`, so a trial can have -one `output_tok_per_s` value for each concurrency. Before comparing trials, -decide on a method to compare their `output_tok_per_s` values across -concurrencies, and keep that method for the whole session. Report the chosen -method via `progress` (see `# Progress`). If `## Additional instructions` -says how to compare trials, use that method. - - During the trials and experimentation aimed at the best performance, you may pick the hardware (the best available within the allowed `dstack` fleets), the model variant (only if `model` has `base`), the serving framework, the @@ -176,6 +167,10 @@ continuing to improve a previous trial's idea, but when that risks settling into a local optimum, search for a substantially different approach rather than tweaking parameters further. + + Before starting trials, analyze the previous sessions' results provided in `previous/` (sessions: {previous}; see `## Previous Sessions`). The @@ -267,7 +262,7 @@ cause. others: ``` -{"resources": {...}, "results": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} +{"resources": {...}, "result": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` 2. In case the task is using node groups (a router group plus worker @@ -275,7 +270,7 @@ cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "results": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} +{"groups": [[{...}], [{...}, {...}], [{...}]], "result": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -303,9 +298,7 @@ cause. [{"cpu": "192", "memory": "2048GB", "disk": "1000GB", "gpu": {"name": "H200", "memory": "141GB", "count": 8}}] ] ``` -- `results`: a list containing one entry for each value of `concurrency` - from `constraints.json`. Each entry contains: - +- `result`: a list of trial result objects, one result object for each value of `concurrency` from `constraints.json`. Each object has fields no other than below: - `concurrency`: the concurrency value for this result. - `context_length`: the largest context the trial's configuration handles at this concurrency, found as described in `## Benchmark`; `null` only @@ -314,7 +307,7 @@ cause. for the structure); `null` only when the benchmark couldn't be done at this concurrency. - `learned`: the major things this trial taught you at this concurrency that - you did not know before it ran. Required for every `results` entry, + you did not know before it ran. Required for every `result` entry, including a failed one. - `failed`: `true` if the configuration did not run or the benchmark broke a constraint such as `max_ttft` or `min_context_length` at this @@ -394,8 +387,8 @@ All verification and benchmark requests must succeed. Record every benchmark using the following structure and field names (values are illustrative): for a trial, in `trials//trial.json` as -`results[concurrency == ].benchmark`; for the final service, as -`final_report.json.results[concurrency == ].benchmark`, where `` is +`result[concurrency == ].benchmark`; for the final service, as +`final_report.json.result[concurrency == ].benchmark`, where `` is a concurrency value from `constraints.json`. ```json @@ -427,17 +420,18 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free invocation. For the final benchmark, use streaming responses. Set -`results[].benchmark.workload.concurrency` to the same value as -`results[].concurrency`. Produce every field of the structure, +`result[concurrency == ].benchmark.workload.concurrency` to the same value as +`result[concurrency == ].concurrency`, where `` +is a concurrency value. Produce every field of the structure, and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and warmup requests. Never invent missing values. After each benchmark, find the largest context the configuration handles by sending real requests, and record it: for a trial, in -`trials//trial.json` as `results[concurrency == ].context_length`; +`trials//trial.json` as `result[concurrency == ].context_length`; for the final service, as -`final_report.json.results[concurrency == ].context_length`, where `` +`final_report.json.result[concurrency == ].context_length`, where `` is a concurrency value. Stopping at the required minimum is not enough. @@ -614,12 +608,12 @@ variable listed under `env` in `constraints.json`. Do not put secret values in `final_report.json`, or print them. Use env references in `final_report.json.service_yaml`; use environment variable -names or redacted values in `final_report.json.results[].benchmark.command`. +names or redacted values in `final_report.json.result[concurrency == ].benchmark.command`, where ``is a concurrency value. # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `results`, and `failure_summary`. +`service_yaml`, `trial`, `base`, `model`, `result`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed trial remained), include exactly: @@ -631,8 +625,7 @@ trial remained), include exactly: - `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `results`: the final service results, one entry for each value of `concurrency` - from `constraints.json`. Each entry contains: +- `result`: list of final service result objects, one object for each value of `concurrency` from `constraints.json`. Each object has fields no other than below: - `concurrency`: the concurrency value for this result - `context_length`: the largest context verified for the final service at this concurrency, as described in `## Benchmark` From 86d337613d4e9580a1b9ef7eff6f97f885ce0c55 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Fri, 13 Nov 2026 14:07:24 +0545 Subject: [PATCH 06/12] Revert to master --- mkdocs/docs/concepts/presets.md | 6 +- .../_internal/cli/models/preset_agent.py | 13 +- .../_internal/cli/services/presets/build.py | 2 - .../_internal/cli/services/presets/output.py | 60 ++----- .../_internal/cli/services/presets/session.py | 100 ++--------- .../_internal/cli/services/presets/verify.py | 30 +--- .../cli/services/presets/workspace.py | 51 +----- .../_internal/core/models/configurations.py | 53 +----- src/dstack/_internal/core/models/presets.py | 37 +---- src/tests/_internal/cli/common.py | 3 +- .../cli/services/presets/test_agent.py | 129 --------------- .../cli/services/presets/test_create.py | 6 +- .../cli/services/presets/test_output.py | 155 +----------------- .../services/presets/test_report_schema.py | 67 ++++---- .../cli/services/presets/test_store.py | 4 +- .../cli/services/presets/test_verify.py | 134 ++------------- .../cli/services/presets/test_workspace.py | 59 ------- .../core/models/test_configurations.py | 43 ----- .../_internal/core/models/test_presets.py | 30 ---- 19 files changed, 109 insertions(+), 873 deletions(-) diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index 4096f11226..b01c63f62b 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -33,9 +33,8 @@ trials: 5 min_context_length: 1048576 max_ttft: 675 -# The numbers of simultaneous requests every benchmark is run at: -# a value, a list, `2,3`, or a doubling range `2..16` -concurrency: 2..16 +# The number of simultaneous requests every benchmark uses +concurrency: 1 # The request shape every benchmark uses (defaults to 1024 and 1024) input_tokens: 10000 @@ -391,6 +390,7 @@ At the same time, it's recommended to create presets using your own agent — ei ## Limitations * Currently, the agent doesn't upload compiled binaries anywhere; patches compile at runtime +* Doesn't support ranges for `concurrency` > Report bugs and request features on [GitHub](https://github.com/dstackai/dstack/issues), and ask questions on [Discord](https://discord.gg/u8SmfwPpMd). diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index c6574aa7bb..812ddce17e 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -9,7 +9,6 @@ ValidationError, ValidatorFunctionWrapHandler, WrapValidator, - model_validator, ) from dstack._internal.core.models.common import CoreModel @@ -54,17 +53,7 @@ class PresetAgentSuccess(CoreModel): base: Annotated[str, Field(min_length=1)] model: Annotated[str, Field(min_length=1)] context_length: PositiveInt - benchmarks: Annotated[list[PresetBenchmark], Field(min_length=1)] - - @model_validator(mode="before") - @classmethod - def wrap_legacy_single_benchmark(cls, data: Any) -> Any: - # Older agents report a single `benchmark`. Wrap it in a list so - # the new CLI can read reports from sessions started before the upgrade. - if isinstance(data, dict) and "benchmark" in data and "benchmarks" not in data: - data = dict(data) - data["benchmarks"] = [data.pop("benchmark")] - return data + benchmark: PresetBenchmark class PresetAgentFailure(CoreModel): diff --git a/src/dstack/_internal/cli/services/presets/build.py b/src/dstack/_internal/cli/services/presets/build.py index 1fa4971158..1c3137f7aa 100644 --- a/src/dstack/_internal/cli/services/presets/build.py +++ b/src/dstack/_internal/cli/services/presets/build.py @@ -32,7 +32,6 @@ def build_preset( base_model: str, repo: str, context_length: int, - benchmarks: list[PresetBenchmark], benchmark: PresetBenchmark, configuration: PresetConfiguration, best_trial: int, @@ -54,7 +53,6 @@ def build_preset( configuration=configuration, created_at=created_at, service=service, - benchmarks=benchmarks, benchmark=benchmark, verified_on=verification_replica_groups, ) diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 3e75d07e50..86f39d54c7 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Any, Callable, Optional, Sequence +from typing import Any, Optional, Sequence from rich.table import Table @@ -198,9 +198,9 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False share = round(100 * shared_prefix_tokens / input_tokens) if share: objective.append(f"prefix={share}%") - sweep = constraints.get("concurrency") or (best or {}).get("concurrency") - if sweep: - objective.append(f"c={_format_concurrency(sweep)}") + concurrency = (best or {}).get("concurrency") or constraints.get("concurrency") + if concurrency: + objective.append(f"c={concurrency}") min_context_length = constraints.get("min_context_length") if verbose and isinstance(min_context_length, int): objective.append(f"ctx>={_format_token_count(min_context_length)}") @@ -211,18 +211,14 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False tps = _format_number(best["tok_s"]) if objective: # Lead with per-user tps: comparable across rows regardless of concurrency. - per_user = best.get("per_user_tok_s_range") - if per_user: - parts.append(f"tps/user={_format_range(per_user, _format_number)}") - # Show the selected concurrency when the sweep has multiple values. - if isinstance(sweep, list) and len(sweep) > 1 and best.get("concurrency"): - parts.append(f"c={best['concurrency']}") + tpot_ms = best.get("tpot_ms") + if isinstance(tpot_ms, (int, float)) and tpot_ms > 0: + parts.append(f"tps/user={_format_number(1000 / tpot_ms)}") if verbose: - tok_s_range = best.get("tok_s_range") or [best["tok_s"]] - parts.append(f"tps={_format_range(tok_s_range, _format_number)}") - ttft_range = best.get("ttft_ms_range") - if ttft_range: - parts.append(f"ttft={_format_range(ttft_range, _format_duration_ms)}") + parts.append(f"tps={tps}") + ttft_ms = best.get("ttft_ms") + if isinstance(ttft_ms, (int, float)): + parts.append(f"ttft={_format_duration_ms(ttft_ms)}") context_length = best.get("context_length") if isinstance(context_length, int): parts.append(f"ctx={_format_token_count(context_length)}") @@ -308,7 +304,7 @@ def format_preset_objective( share = round(100 * workload.shared_prefix_tokens / workload.input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={_format_concurrency([b.workload.concurrency for b in preset.benchmarks])}") + parts.append(f"c={workload.concurrency}") return f"[secondary]{' '.join(parts)}[/]" @@ -327,7 +323,7 @@ def _format_creation_objective( share = round(100 * (configuration.shared_prefix_tokens or 0) / input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={_format_concurrency(configuration.concurrency or [workload.concurrency])}") + parts.append(f"c={configuration.concurrency or workload.concurrency}") if verbose and configuration.min_context_length is not None: parts.append(f"ctx>={_format_token_count(configuration.min_context_length)}") if verbose and configuration.max_ttft is not None: @@ -349,21 +345,15 @@ def _breaches_constraints(preset: AnyStoredPreset) -> bool: def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) -> str: - # Show benchmark ranges across the sweep and identify the selected concurrency. - benchmarks = preset.benchmarks + benchmark = preset.benchmark + metrics = benchmark.metrics parts = [ - "tps/user=" - + _format_range([b.effective_per_user_tok_per_s for b in benchmarks], _format_number) + f"tps/user={_format_number(benchmark.effective_per_user_tok_per_s)}", ] - if len(benchmarks) > 1: - parts.append(f"c={preset.benchmark.workload.concurrency}") if verbose: - parts.append( - "tps=" - + _format_range([b.effective_output_tok_per_s for b in benchmarks], _format_number) - ) + parts.append(f"tps={_format_number(benchmark.effective_output_tok_per_s)}") parts += [ - "ttft=" + _format_range([b.metrics.ttft_ms.p50 for b in benchmarks], _format_duration_ms), + f"ttft={_format_duration_ms(metrics.ttft_ms.p50)}", f"ctx={_format_token_count(preset.context_length)}", ] text = " ".join(parts) @@ -372,20 +362,6 @@ def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) - return text -def _format_range(values: list[float], fmt: Callable[[float], str]) -> str: - low, high = fmt(min(values)), fmt(max(values)) - return low if low == high else f"{low}..{high}" - - -def _format_concurrency(value: Any) -> str: - """Format concurrency values for display, for example: 8, 2,3, or 2..16.""" - if not isinstance(value, list): - return str(value) - if len(value) > 1 and all(b == a * 2 for a, b in zip(value, value[1:])): - return f"{value[0]}..{value[-1]}" - return ",".join(str(v) for v in value) - - def _format_duration_ms(value: float) -> str: # 999.6 rounds to 1000, which must read as 1s rather than 1000ms. if value < 999.5: diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index e56eeadd28..9ce77213e3 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -529,12 +529,9 @@ def list_preset_sessions() -> list[dict[str, Any]]: entry["id"] = path.name entry["name"] = claimed_session_name(state) entry["status"] = status - constraints = _read_session_constraints(path) - entry["trials"] = _summarize_session_trials( - path / TRIALS_DIRNAME, max_ttft=constraints.get("max_ttft") - ) + entry["trials"] = _summarize_session_trials(path / TRIALS_DIRNAME) entry["verification"] = _read_last_session_verification(path / SERVICE_DIRNAME) - entry["constraints"] = constraints + entry["constraints"] = _read_session_constraints(path) entries.append(entry) return entries @@ -587,9 +584,7 @@ def _read_last_session_verification(path: Path) -> Optional[dict[str, Any]]: return {"status": "verifying"} -def _summarize_session_trials( - path: Path, *, max_ttft: Optional[float] = None -) -> Optional[dict[str, Any]]: +def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: # TODO: Refactor this crap - must be explicit what is this and where is it used; also dicts are prohibited in dstack repo """A trial directory without `trial.json` is still in flight and is not counted.""" @@ -610,25 +605,35 @@ def _summarize_session_trials( gpu: Optional[str] = None for record in records: count += 1 + benchmark = record.get("benchmark") failed.append(bool(record.get("failed"))) record_gpu = _format_trial_gpu(record) if record_gpu: gpu = record_gpu - point = _select_trial_point(record, max_ttft) - if point is None: + if not isinstance(benchmark, dict): + series.append(None) + continue + metrics = benchmark.get("metrics") or {} + workload = benchmark.get("workload") or {} + duration = metrics.get("duration_seconds") + tokens = metrics.get("total_output_tokens") + if not isinstance(duration, (int, float)) or duration <= 0: series.append(None) continue - tok_s, metrics, workload, sweep = point + if not isinstance(tokens, (int, float)): + series.append(None) + continue + tok_s = tokens / duration series.append(tok_s) # A failed trial keeps its benchmark — it is what the next trial learns # from — but it is not a candidate for best, and promoting one would put # a configuration that broke a constraint at the top of the listing. if record.get("failed"): if best_failed is None or tok_s > best_failed["tok_s"]: - best_failed = _trial_entry(tok_s, record, metrics, workload, sweep, record_gpu) + best_failed = _trial_entry(tok_s, record, metrics, workload, record_gpu) continue if best is None or tok_s > best["tok_s"]: - best = _trial_entry(tok_s, record, metrics, workload, sweep, record_gpu) + best = _trial_entry(tok_s, record, metrics, workload, record_gpu) return { "count": count, "best": best, @@ -639,60 +644,11 @@ def _summarize_session_trials( } -def _select_trial_point( - record: dict[str, Any], max_ttft: Optional[float] -) -> Optional[tuple[float, dict[str, Any], dict[str, Any], list[dict[str, Any]]]]: - """Select the benchmark point used to judge a trial. - - Choose the highest-throughput point whose p50 TTFT meets `max_ttft`. - If no point meets the limit, choose the point with the lowest p50 TTFT. - Older trial records may contain a single `benchmark` instead of `benchmarks`. - - Return `(tok_s, metrics, workload, sweep)`, where `metrics` and `workload` - describe the selected point and `sweep` contains metrics for all usable - points. Return `None` if no point has usable throughput data. - """ - benchmarks = record.get("benchmarks") - if not isinstance(benchmarks, list): - benchmarks = [record.get("benchmark")] - points: list[tuple[bool, float, Any, dict[str, Any], dict[str, Any]]] = [] - for benchmark in benchmarks: - if not isinstance(benchmark, dict): - continue - metrics = benchmark.get("metrics") or {} - workload = benchmark.get("workload") or {} - duration = metrics.get("duration_seconds") - tokens = metrics.get("total_output_tokens") - if not isinstance(duration, (int, float)) or duration <= 0: - continue - if not isinstance(tokens, (int, float)): - continue - ttft = (metrics.get("ttft_ms") or {}).get("p50") - within = not isinstance(max_ttft, (int, float)) or ( - isinstance(ttft, (int, float)) and ttft <= max_ttft - ) - points.append((within, tokens / duration, ttft, metrics, workload)) - if not points: - return None - sweep = [p[3] for p in points] - passing = [p for p in points if p[0]] - if passing: - _, tok_s, _, metrics, workload = max(passing, key=lambda p: p[1]) - else: - # `ttft` may be missing in a raw record; sort those last. - _, tok_s, _, metrics, workload = min( - points, - key=lambda p: (p[2] if isinstance(p[2], (int, float)) else float("inf"), -p[1]), - ) - return tok_s, metrics, workload, sweep - - def _trial_entry( tok_s: float, record: dict[str, Any], metrics: dict[str, Any], workload: dict[str, Any], - sweep: list[dict[str, Any]], gpu: Optional[str], ) -> dict[str, Any]: ttft = (metrics.get("ttft_ms") or {}).get("p50") @@ -703,32 +659,12 @@ def _trial_entry( "tok_s": tok_s, "tpot_ms": tpot if isinstance(tpot, (int, float)) and tpot > 0 else None, "ttft_ms": ttft if isinstance(ttft, (int, float)) else None, - # Store the lowest and highest throughput and TTFT values across the trial's - # concurrency points. For example, throughput values 132, 237, and 380 - # tokens/sec become [132, 380]. One value becomes [value, value]; no values - # becomes None. - "tok_s_range": _range([m["total_output_tokens"] / m["duration_seconds"] for m in sweep]), - "per_user_tok_s_range": _range( - [1000 / v for v in (_latency(m, "tpot_ms", "mean") for m in sweep) if v] - ), - "ttft_ms_range": _range( - [v for v in (_latency(m, "ttft_ms", "p50") for m in sweep) if v is not None] - ), "context_length": context_length if isinstance(context_length, int) else None, "concurrency": workload.get("concurrency"), "gpu": gpu, } -def _latency(metrics: dict[str, Any], name: str, stat: str) -> Optional[float]: - value = (metrics.get(name) or {}).get(stat) - return value if isinstance(value, (int, float)) and value >= 0 else None - - -def _range(values: list[float]) -> Optional[list[float]]: - return [min(values), max(values)] if values else None - - def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: """A trial records its hardware in one of two formats, as described in `system_prompt.md`: `{"resources": {...}}` without node groups, and diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index c473a5b4b4..643522c503 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -31,7 +31,6 @@ PresetBenchmark, PresetVerificationReplicaGroup, PresetWorkload, - select_benchmark, ) from dstack._internal.core.models.runs import JobStatus, Run, RunStatus @@ -100,10 +99,6 @@ def build_verified_preset( _check_report_answers_request(report, preset_configuration) if service.model is None or service.model.name != preset_configuration.model.api_model_name: raise CLIError("The agent's final service model name does not match the requested model") - benchmarks = sorted( - (_normalized_benchmark(b, preset_configuration) for b in report.benchmarks), - key=lambda b: b.workload.concurrency, - ) return build_preset( name=name, service=_portable_service( @@ -119,8 +114,7 @@ def build_verified_preset( # it cannot be confused with the service's client-facing model name. repo=report.model, context_length=report.context_length, - benchmarks=benchmarks, - benchmark=select_benchmark(benchmarks, preset_configuration.max_ttft), + benchmark=_normalized_benchmark(report.benchmark, preset_configuration), best_trial=report.trial, configuration=preset_configuration, preset_id=preset_id, @@ -145,11 +139,9 @@ def _check_report_answers_request( report: PresetAgentSuccess, configuration: PresetConfiguration ) -> None: """The report must answer what the configuration asked: the same benchmark - workload at every concurrency of the sweep, and the requested model — exactly - when it was exact, any variant of the base otherwise.""" - for benchmark in report.benchmarks: - _check_workload_answers_request(benchmark.workload, configuration) - _check_sweep_answers_request(report.benchmarks, configuration) + workload, and the requested model — exactly when it was exact, any variant of + the base otherwise.""" + _check_workload_answers_request(report.benchmark.workload, configuration) if configuration.model.allows_variant_selection: if report.base != configuration.model.api_model_name: raise CLIError("The agent's final report base does not match the requested model") @@ -180,18 +172,10 @@ def _check_workload_answers_request( f"The agent's final benchmark shared prefix of {workload.shared_prefix_tokens}" f" tokens does not match the requested {shared_prefix_tokens}" ) - - -def _check_sweep_answers_request( - benchmarks: Sequence[PresetBenchmark], configuration: PresetConfiguration -) -> None: - """One benchmark per requested concurrency, no more and no fewer: a missing - point would hide where the service stops meeting the constraints.""" - reported = sorted(b.workload.concurrency for b in benchmarks) - if configuration.concurrency is not None and reported != configuration.concurrency: + if configuration.concurrency is not None and workload.concurrency != configuration.concurrency: raise CLIError( - f"The agent's final benchmark concurrencies {reported} do not match the" - f" requested {configuration.concurrency}" + f"The agent's final benchmark concurrency of {workload.concurrency} does not match the" + f" requested concurrency of {configuration.concurrency}" ) diff --git a/src/dstack/_internal/cli/services/presets/workspace.py b/src/dstack/_internal/cli/services/presets/workspace.py index 7d5b609cf6..97ade98b38 100644 --- a/src/dstack/_internal/cli/services/presets/workspace.py +++ b/src/dstack/_internal/cli/services/presets/workspace.py @@ -10,7 +10,7 @@ from contextlib import suppress from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional, Sequence +from typing import Optional, Sequence from dstack._internal.cli.models.preset_agent import PresetSessionWorkspace from dstack._internal.cli.services.presets.session import ( @@ -300,7 +300,7 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: for name in (_CONSTRAINTS_FILENAME, _FINAL_REPORT_FILENAME): if (source_root / name).is_file(): target_root.mkdir(parents=True, exist_ok=True) - _copy_record(source_root / name, target_root / name) + shutil.copyfile(source_root / name, target_root / name) copied = True for group, filenames in ( (TRIALS_DIRNAME, ("trial.json", "task.dstack.yml")), @@ -316,7 +316,7 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: for name in filenames: if (record_dir / name).is_file(): target_dir.mkdir(parents=True, exist_ok=True) - _copy_record(record_dir / name, target_dir / name) + shutil.copyfile(record_dir / name, target_dir / name) copied = True patches = record_dir / "patches" if group == TRIALS_DIRNAME and patches.is_dir(): @@ -325,51 +325,6 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: return copied -# The records an agent reads through `previous/`, upgraded on copy so a -# session recorded before concurrency sweeps reads in the format the agent -# writes. The originals stay as they were written. -_UPGRADED_RECORDS = (_CONSTRAINTS_FILENAME, _FINAL_REPORT_FILENAME, "trial.json") - - -def _copy_record(source: Path, target: Path) -> None: - if source.name not in _UPGRADED_RECORDS: - shutil.copyfile(source, target) - return - try: - data = json.loads(source.read_text(encoding="utf-8")) - except (OSError, ValueError): - # A torn or foreign file is still the agent's to read as written. - shutil.copyfile(source, target) - return - upgraded = _upgrade_pre_sweep_record(source.name, data) - if upgraded is data: - shutil.copyfile(source, target) - return - target.write_text(json.dumps(upgraded, indent=2) + "\n", encoding="utf-8") - - -def _upgrade_pre_sweep_record(name: str, data: Any) -> Any: - # Before the sweep, `constraints.json` held one `concurrency` and trial and - # final records held one `benchmark`: that one point is the whole sweep. - if not isinstance(data, dict): - return data - if name == _CONSTRAINTS_FILENAME: - concurrency = data.get("concurrency") - if isinstance(concurrency, int): - return {**data, "concurrency": [concurrency]} - return data - if "benchmark" not in data or "benchmarks" in data: - return data - upgraded = {} - for key, value in data.items(): - if key == "benchmark": - # `null` means no benchmark could be done; it stays `null`. - upgraded["benchmarks"] = None if value is None else [value] - else: - upgraded[key] = value - return upgraded - - def _install_skills(target_dir: Path) -> None: source_dir = _get_skills_dir() for skill_name in _SKILL_NAMES: diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index bb2caaafa3..6cf8e423ff 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -1774,14 +1774,12 @@ class PresetConfiguration( ), ] = None concurrency: Annotated[ - Optional[list[PositiveInt]], + Optional[PositiveInt], Field( - min_length=1, description=( - "The numbers of simultaneous requests every benchmark is run at during" - " preset creation: a single value, a list, `2,3`, or a doubling range" - " `2..16` (2, 4, 8, 16). Required for creation" - ), + "The number of simultaneous requests used for benchmarks during preset" + " creation. Required for creation" + ) ), ] = None input_tokens: Annotated[ @@ -1868,22 +1866,6 @@ def effective_output_tokens(self) -> int: def effective_baseline(self) -> bool: return self.baseline if self.baseline is not None else DEFAULT_BASELINE - @field_validator( - "concurrency", - mode="before", - json_schema_input_type=Union[PositiveInt, str, list[PositiveInt]], - ) - @classmethod - def parse_concurrency(cls, value: Any) -> Any: - return _parse_concurrency_sweep(value) - - @field_validator("concurrency") - @classmethod - def sort_concurrency(cls, value: Optional[list[int]]) -> Optional[list[int]]: - # Ascending and unique, whichever way it was written: the sweep is compared - # to what the agent reports, and the report is ordered the same way. - return sorted(set(value)) if value is not None else None - @field_validator("dataset") @classmethod def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]: @@ -1964,33 +1946,6 @@ def validate_prompt(cls, value: Any) -> Any: return value -def _parse_concurrency_sweep(value: Any) -> Any: - # `8`, `2,3`, `[2, 3]`, or `2..16`. A range doubles from its start and must - # land exactly on its end, so `3..16` is rejected rather than stopping at 12 - # or overshooting to 24. Element validation is left to the field type. - if value is None or isinstance(value, (bool, list)): - return value - if isinstance(value, int): - return [value] - if not isinstance(value, str): - return value - text = value.replace(" ", "") - if ".." not in text: - return text.split(",") - start_text, _, end_text = text.partition("..") - if not start_text.isdigit() or not end_text.isdigit(): - raise ValueError("a concurrency range needs both ends, e.g. `2..16`") - start, end = int(start_text), int(end_text) - if start < 1 or end < start: - raise ValueError("a concurrency range must go from a positive start up to its end") - values = [start] - while values[-1] < end: - values.append(values[-1] * 2) - if values[-1] != end: - raise ValueError(f"concurrency range end {end} is not reachable by doubling from {start}") - return values - - def _validate_model(value: Any, *, field: str) -> str: if not isinstance(value, str) or not value.strip(): raise ValueError(f"Preset model {field} must be a non-empty string") diff --git a/src/dstack/_internal/core/models/presets.py b/src/dstack/_internal/core/models/presets.py index 1258dcb1b1..95e87fb29f 100644 --- a/src/dstack/_internal/core/models/presets.py +++ b/src/dstack/_internal/core/models/presets.py @@ -128,22 +128,6 @@ def validate_metrics(self) -> Self: return self -def select_benchmark( - benchmarks: Sequence[PresetBenchmark], max_ttft: Optional[int] -) -> PresetBenchmark: - """Picks the sweep point a preset is judged by. - - Rule 1: Keep the points whose p50 TTFT is within `max_ttft`. - Rule 2: If any are kept, pick the one with the highest output throughput. - Rule 3: If none are kept, pick the one with the lowest p50 TTFT, so the - preset records how close it got to the cap rather than how fast it ran. - Rule 4: If two points tie on TTFT in rule 3, pick the higher throughput.""" - passing = [b for b in benchmarks if max_ttft is None or b.metrics.ttft_ms.p50 <= max_ttft] - if passing: - return max(passing, key=lambda b: b.effective_output_tok_per_s) - return min(benchmarks, key=lambda b: (b.metrics.ttft_ms.p50, -b.effective_output_tok_per_s)) - - class PresetVerificationReplicaGroup(CoreModel): # The service replica group this was measured for. name: str @@ -169,35 +153,16 @@ class PortablePreset(CoreModel): # stored as-is. `files` paths are stored relative to the preset directory, # absolute after load. service: ServiceConfiguration - benchmarks: Annotated[List[PresetBenchmark], Field(min_length=1)] - """The final service benchmarked at every concurrency of the sweep, ascending.""" benchmark: PresetBenchmark - """The one of `benchmarks` the preset is judged and listed by, chosen by - `select_benchmark`. Kept as its own field so clients that predate the sweep - keep reading it.""" # The hardware it was verified on: the actual resources of every running # replica, by service replica group. verified_on: List[PresetVerificationReplicaGroup] - @model_validator(mode="before") - @classmethod - def wrap_legacy_single_benchmark(cls, data: Any) -> Any: - # Before the sweep, a preset was benchmarked at one concurrency and - # stored it as `benchmark` alone. That one point is its whole sweep. - if isinstance(data, dict) and "benchmark" in data and "benchmarks" not in data: - data = {**data, "benchmarks": [data["benchmark"]]} - return data - @model_validator(mode="after") def validate_artifact(self) -> Self: service = self.service if service.model is None: raise ValueError("preset service must specify model") - concurrencies = [b.workload.concurrency for b in self.benchmarks] - if concurrencies != sorted(set(concurrencies)): - raise ValueError("preset benchmarks must be at unique, ascending concurrencies") - if self.benchmark not in self.benchmarks: - raise ValueError("preset benchmark must be one of its benchmarks") if any(group.resources is None for group in service.replica_groups): raise ValueError("preset service must specify resources") for field in PRESET_EXCLUDED_FIELDS: @@ -339,7 +304,7 @@ class PresetConstraints(CoreModel): min_context_length: PositiveInt max_ttft: PositiveInt trials_num: PositiveInt - concurrency: Annotated[list[PositiveInt], Field(min_length=1)] + concurrency: PositiveInt baseline: bool fleets: Annotated[list[str], Field(min_length=1)] env: list[str] diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index 144d2e1a2b..add3afc533 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -171,7 +171,6 @@ def get_preset( ), context_length=context_length, best_trial=1, - benchmarks=[get_preset_benchmark()], benchmark=get_preset_benchmark(), verified_on=[ PresetVerificationReplicaGroup(name=DEFAULT_REPLICA_GROUP_NAME, replicas=[resources]) @@ -279,7 +278,7 @@ def get_successful_preset_report(run: Run) -> PresetAgentSuccess: "base": "Qwen/Qwen3.5-27B", "model": "community/Qwen3.5-27B-GPTQ-Int4", "context_length": 32768, - "benchmarks": [get_preset_benchmark()], + "benchmark": get_preset_benchmark(), } ) diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index 4167331685..e55c3fd013 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -1642,9 +1642,6 @@ def test_counts_records_even_when_trials_share_a_task(self, tmp_path): "tok_s": 2300.0, "tpot_ms": None, "ttft_ms": None, - "tok_s_range": [2300.0, 2300.0], - "per_user_tok_s_range": None, - "ttft_ms_range": None, "context_length": None, "concurrency": 8, "gpu": "A40:48GB:1", @@ -1687,132 +1684,6 @@ def test_a_trial_still_in_flight_is_not_counted(self, tmp_path): assert summary["count"] == 1 assert summary["series"] == [2000.0] - def test_judges_a_sweep_by_its_fastest_point_within_max_ttft(self, tmp_path): - def point(concurrency, tok_s, ttft_p50): - return { - "workload": {"concurrency": concurrency}, - "metrics": { - "duration_seconds": 1.0, - "total_output_tokens": tok_s, - "ttft_ms": {"p50": ttft_p50}, - }, - } - - record = { - "resources": {}, - # Throughput rises with concurrency, but the top point breaches the cap. - "benchmarks": [point(2, 500.0, 100.0), point(4, 800.0, 300.0), point(8, 900.0, 900.0)], - } - trials_dir = _write_trials(tmp_path, [record]) - - summary = _summarize_session_trials(trials_dir, max_ttft=500) - assert summary is not None - - assert summary["series"] == [800.0] - best = summary["best"] - assert best is not None - assert best["concurrency"] == 4 - assert best["ttft_ms"] == 300.0 - # The selected point is c=4; the ranges still cover every point. - assert best["ttft_ms_range"] == [100.0, 900.0] - - def test_ranges_come_from_the_selected_trial_only(self, tmp_path): - def point(concurrency, tokens, ttft, tpot): - return { - "workload": {"concurrency": concurrency}, - "metrics": { - "duration_seconds": 1.0, - "total_output_tokens": tokens, - "ttft_ms": {"p50": ttft}, - "tpot_ms": {"mean": tpot}, - }, - } - - slow = {"benchmarks": [point(4, 100.0, 50.0, 10.0), point(8, 200.0, 80.0, 20.0)]} - fast = {"benchmarks": [point(4, 300.0, 400.0, 30.0), point(8, 600.0, 450.0, 40.0)]} - trials_dir = _write_trials(tmp_path, [slow, fast]) - - summary = _summarize_session_trials(trials_dir, max_ttft=500) - assert summary is not None - - best = summary["best"] - assert best is not None - assert best["concurrency"] == 8 - assert best["ttft_ms_range"] == [400.0, 450.0] - assert best["per_user_tok_s_range"] == [25.0, 1000 / 30.0] - - def test_a_partial_sweep_ranges_over_its_usable_points(self, tmp_path): - usable = { - "workload": {"concurrency": 4}, - "metrics": { - "duration_seconds": 1.0, - "total_output_tokens": 100.0, - "ttft_ms": {"p50": 200.0}, - "tpot_ms": {"mean": 20.0}, - }, - } - broken = {"workload": {"concurrency": 8}, "metrics": {"duration_seconds": 0}} - trials_dir = _write_trials(tmp_path, [{"benchmarks": [usable, broken]}]) - - summary = _summarize_session_trials(trials_dir, max_ttft=500) - assert summary is not None - - best = summary["best"] - assert best is not None - assert best["ttft_ms_range"] == [200.0, 200.0] - assert best["per_user_tok_s_range"] == [50.0, 50.0] - - def test_a_pre_sweep_record_has_a_single_point_range(self, tmp_path): - record = { - "benchmark": { - "workload": {"concurrency": 8}, - "metrics": { - "duration_seconds": 1.0, - "total_output_tokens": 100.0, - "ttft_ms": {"p50": 240.0}, - "tpot_ms": {"mean": 25.0}, - }, - } - } - trials_dir = _write_trials(tmp_path, [record]) - - summary = _summarize_session_trials(trials_dir, max_ttft=500) - assert summary is not None - - best = summary["best"] - assert best is not None - assert best["ttft_ms_range"] == [240.0, 240.0] - - def test_judges_a_sweep_that_breached_everywhere_by_its_closest_point(self, tmp_path): - record = { - "benchmarks": [ - { - "workload": {"concurrency": 4}, - "metrics": { - "duration_seconds": 1.0, - "total_output_tokens": 800.0, - "ttft_ms": {"p50": 700.0}, - }, - }, - { - "workload": {"concurrency": 8}, - "metrics": { - "duration_seconds": 1.0, - "total_output_tokens": 900.0, - "ttft_ms": {"p50": 900.0}, - }, - }, - ] - } - trials_dir = _write_trials(tmp_path, [record]) - - summary = _summarize_session_trials(trials_dir, max_ttft=500) - assert summary is not None - - best = summary["best"] - assert best is not None - assert best["concurrency"] == 4 - def test_a_torn_result_copy_reads_as_still_in_flight(self, tmp_path): trials_dir = _write_trials(tmp_path, []) (trials_dir / "1").mkdir(parents=True) diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index c4c4dcf09e..c6203023f4 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -863,7 +863,7 @@ def test_renders_defaults_for_the_optional_fields(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": [8], + "concurrency": 8, "input_tokens": 1024, "output_tokens": 1024, "shared_prefix_tokens": 0, @@ -897,7 +897,7 @@ def test_renders_custom_dataset_without_request_shape(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": [8], + "concurrency": 8, "dataset": "spec_bench", "baseline": True, "fleets": ["gpu-fleet"], @@ -927,7 +927,7 @@ def test_renders_configured_values(self): assert data["model"] == {"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"} assert data["min_context_length"] == 32768 assert data["trials_num"] == 10 - assert data["concurrency"] == [16] + assert data["concurrency"] == 16 assert data["fleets"] == ["a", "b"] diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index 749b971b5c..e5e963388b 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -32,7 +32,6 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): ttft.mean = 8148.3 ttft.p50 = 8151.4 ttft.p99 = 8334.2 - preset.benchmarks = [preset.benchmark] output = output_module.format_preset_benchmark(preset, verbose=True) @@ -43,28 +42,6 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): assert "ttft=8.15s" in output assert "e+03" not in output - def test_shows_a_sweep_as_ranges_and_names_the_selected_point(self): - preset = get_preset() - points = [] - for concurrency, tpot, ttft, tokens in ((4, 30.0, 425.0, 1024), (16, 52.0, 861.0, 4096)): - point = preset.benchmark.model_copy(deep=True) - point.workload.concurrency = concurrency - point.metrics.tpot_ms.mean = tpot - point.metrics.ttft_ms.p50 = ttft - point.metrics.total_output_tokens = tokens - points.append(point) - preset.benchmarks = points - preset.benchmark = points[1] - - output = output_module.format_preset_benchmark(preset, verbose=True) - - assert output == "tps/user=19.2..33.3 c=16 tps=21.1..84.2 ttft=425ms..861ms ctx=32K" - - def test_shows_single_values_without_a_sweep(self): - output = output_module.format_preset_benchmark(get_preset()) - - assert output == "tps/user=133 ttft=108ms ctx=32K" - class TestFormatPresetObjective: def test_shows_the_requested_shared_prefix(self): @@ -163,7 +140,7 @@ def test_prints_submitted_column(self, monkeypatch): assert "2 months ago" in output.getvalue() -def _session_row(session: dict, *, verbose: bool = False) -> dict: +def _session_row(session: dict) -> dict: table = Table(box=None) for column in ( "BASE", @@ -177,7 +154,7 @@ def _session_row(session: dict, *, verbose: bool = False) -> dict: "SUBMITTED", ): table.add_column(column) - _add_session(table, session, verbose=verbose) + _add_session(table, session) return { column.header: "".join(str(cell) for cell in column._cells) for column in table.columns } @@ -194,7 +171,7 @@ def test_shows_progress_after_status_and_best_benchmark(self): "count": 2, "best": { "tok_s": 2339.0, - "per_user_tok_s_range": [292.0, 292.0], + "tpot_ms": 3.42, "concurrency": 8, "gpu": "A40:48GB:1", }, @@ -225,84 +202,6 @@ def test_shows_the_shared_prefix_when_the_workload_has_one(self): assert row["CONSTRAINTS"] == ("[secondary]io=8K/1K prefix=90% c=162[/]") - def test_shows_a_sweep_and_the_point_the_best_trial_is_judged_by(self): - row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "constraints": {"dataset": "spec_bench", "concurrency": [2, 4, 8, 16]}, - "trials": { - "count": 1, - "best": { - "tok_s": 2339.0, - "per_user_tok_s_range": [292.0, 292.0], - "concurrency": 8, - "gpu": None, - }, - }, - } - ) - - assert row["CONSTRAINTS"] == "[secondary]data=spec_bench c=2..16[/]" - assert row["BENCHMARK"].startswith("tps/user=292 c=8") - - def test_shows_a_sweep_session_as_ranges(self): - row = _session_row( - { - "id": "6ac0bec5", - "status": "interrupted", - "constraints": {"concurrency": [4, 8, 16]}, - "trials": { - "count": 1, - "best": { - "tok_s": 380.0, - "tok_s_range": [132.0, 380.0], - "per_user_tok_s_range": [24.3, 33.3], - "ttft_ms_range": [425.0, 861.0], - "context_length": 98304, - "concurrency": 16, - }, - }, - }, - verbose=True, - ) - - assert row["BENCHMARK"] == ( - "tps/user=24.3..33.3 c=16 tps=132..380 ttft=425ms..861ms ctx=96K" - ) - - def test_a_failed_sweep_keeps_its_ranges_and_its_mark(self): - row = _session_row( - { - "id": "6ac0bec5", - "status": "failed", - "constraints": {"concurrency": [4, 8]}, - "trials": { - "count": 1, - "best": None, - "best_failed": { - "tok_s": 200.0, - "per_user_tok_s_range": [25.0, 33.3], - "ttft_ms_range": [900.0, 1100.0], - "concurrency": 4, - }, - }, - } - ) - - assert row["BENCHMARK"] == "[secondary]*tps/user=25..33.3 c=4 ttft=900ms..1.1s[/]" - - def test_shows_a_sweep_that_does_not_double_as_a_list(self): - row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "constraints": {"dataset": "spec_bench", "concurrency": [2, 3]}, - } - ) - - assert row["CONSTRAINTS"] == "[secondary]data=spec_bench c=2,3[/]" - def test_shows_the_dataset_for_a_session_with_a_custom_dataset(self): row = _session_row( { @@ -612,8 +511,8 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): "best": None, "best_failed": { "tok_s": 100.0, - "per_user_tok_s_range": [29.1, 29.1], - "ttft_ms_range": [4300.0, 4300.0], + "tpot_ms": 34.4, + "ttft_ms": 4300.0, "context_length": 1048576, "concurrency": 4, "gpu": "MI300X:192GB:1", @@ -628,50 +527,6 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): # styling is absent or invisible to the reader. assert "*tps/user" in row["BENCHMARK"] - def test_a_sweep_trial_record_reaches_the_row_as_ranges(self, tmp_path): - from dstack._internal.cli.services.presets.session import _summarize_session_trials - - # The summary writes the keys the row reads; only both together prove it. - def point(concurrency, tokens, ttft, tpot): - return { - "workload": {"concurrency": concurrency}, - "metrics": { - "duration_seconds": 1.0, - "total_output_tokens": tokens, - "ttft_ms": {"p50": ttft}, - "tpot_ms": {"mean": tpot}, - }, - } - - trials_dir = _write_trials( - tmp_path, - [ - { - "context_length": 98304, - "benchmarks": [ - point(4, 132.0, 425.0, 30.0), - point(8, 237.0, 691.0, 34.0), - point(16, 380.0, 861.0, 41.2), - ], - } - ], - ) - - trials = _summarize_session_trials(trials_dir, max_ttft=2000) - row = _session_row( - { - "id": "6ac0bec5", - "status": "interrupted", - "constraints": {"concurrency": [4, 8, 16], "max_ttft": 2000}, - "trials": trials, - }, - verbose=True, - ) - - assert row["BENCHMARK"] == ( - "tps/user=24.3..33.3 c=16 tps=132..380 ttft=425ms..861ms ctx=96K" - ) - class TestFailedTrialSpark: def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): diff --git a/src/tests/_internal/cli/services/presets/test_report_schema.py b/src/tests/_internal/cli/services/presets/test_report_schema.py index 0e9eb437e7..9298e06d2a 100644 --- a/src/tests/_internal/cli/services/presets/test_report_schema.py +++ b/src/tests/_internal/cli/services/presets/test_report_schema.py @@ -19,7 +19,7 @@ "base": None, "model": None, "context_length": None, - "benchmarks": None, + "benchmark": None, "failure_summary": None, } # Keywords the OpenAI strict structured-output mode rejects. @@ -50,15 +50,6 @@ def _walk(node): yield from _walk(item) -class TestGetReportJsonSchema: - def test_offers_only_the_sweep_shape(self): - # The legacy `benchmark` is accepted at parse only; agents are never offered it. - properties = get_report_json_schema()["properties"] - - assert "benchmarks" in properties - assert "benchmark" not in properties - - class TestGetStrictReportJsonSchema: def test_every_object_requires_all_of_its_properties(self): objects = [ @@ -115,33 +106,31 @@ def test_keeps_the_agent_facing_shape(self): "base": "Qwen/Qwen2.5-0.5B-Instruct", "model": "Qwen/Qwen2.5-0.5B-Instruct", "context_length": 2048, - "benchmarks": [ - { - "tool": "vllm.bench serve", - "tool_version": "0.27.0", - "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", - "workload": { - "api": "chat_completions", - "dataset": None, - "num_requests": 32, - "input_tokens": 128, - "output_tokens": 64, - "concurrency": 1, - "shared_prefix_tokens": 0, - }, - "metrics": { - "successful_requests": 32, - "failed_requests": 0, - "duration_seconds": 40.5, - "total_input_tokens": 4096, - "total_output_tokens": 2048, - "output_tok_per_s": 50.6, - "per_user_tok_per_s": 50.6, - "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, - "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, - }, - } - ], + "benchmark": { + "tool": "vllm.bench serve", + "tool_version": "0.27.0", + "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", + "workload": { + "api": "chat_completions", + "dataset": None, + "num_requests": 32, + "input_tokens": 128, + "output_tokens": 64, + "concurrency": 1, + "shared_prefix_tokens": 0, + }, + "metrics": { + "successful_requests": 32, + "failed_requests": 0, + "duration_seconds": 40.5, + "total_input_tokens": 4096, + "total_output_tokens": 2048, + "output_tok_per_s": 50.6, + "per_user_tok_per_s": 50.6, + "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, + "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, + }, + }, }, PresetAgentSuccess, ), @@ -157,5 +146,5 @@ def test_reports_codex_produced_against_the_schema_parse(self, data, expected): assert isinstance(report, expected) if isinstance(report, PresetAgentSuccess): assert isinstance(report.service_yaml, ServiceConfiguration) - assert report.benchmarks[0].workload.dataset is None - assert report.benchmarks[0].workload.shared_prefix_tokens == 0 + assert report.benchmark.workload.dataset is None + assert report.benchmark.workload.shared_prefix_tokens == 0 diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index 0dbcc66929..fb774a68ff 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -383,7 +383,7 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert preset.configuration.input_tokens == 10000 assert preset.configuration.output_tokens == 1500 assert preset.configuration.shared_prefix_tokens == 0 - assert preset.configuration.concurrency == [4] + assert preset.configuration.concurrency == 4 # `priority` is not an excluded field, so the regrouping keeps it. assert preset.service.priority == 0 assert [group.name for group in preset.verified_on] == [ @@ -392,8 +392,6 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert get_or_error(preset.verified_on[0].replicas[0].gpu).name == ["MI300X"] # The old format never recorded a dataset name for a synthetic workload. assert preset.benchmark.workload.dataset is None - # One concurrency, so the one benchmark is the whole sweep. - assert preset.benchmarks == [preset.benchmark] assert store.list() == [preset] def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index 97e2ab1321..c7a4d9ea80 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -4,11 +4,7 @@ import pytest from pydantic import ValidationError -from dstack._internal.cli.models.preset_agent import ( - AnyPresetAgentResult, - PresetAgentFailure, - PresetAgentSuccess, -) +from dstack._internal.cli.models.preset_agent import AnyPresetAgentResult, PresetAgentSuccess from dstack._internal.cli.services.presets.agent import ( PresetAgentProcessOutput, ) @@ -65,7 +61,6 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): base_model="Qwen/Qwen3.5-27B", repo="community/Qwen3.5-27B-GPTQ-Int4", context_length=32768, - benchmarks=[get_preset_benchmark()], benchmark=get_preset_benchmark(), configuration=configuration, best_trial=1, @@ -81,12 +76,12 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): assert preset.configuration.spot_policy is None assert "secret" not in preset.model_dump_json() - def test_successful_report_requires_benchmarks(self): + def test_successful_report_requires_benchmark(self): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() - data.pop("benchmarks") + data.pop("benchmark") - with pytest.raises(ValidationError, match="benchmarks"): + with pytest.raises(ValidationError, match="benchmark"): validate_extra_ignore(AnyPresetAgentResult, data) def test_builds_portable_self_contained_preset(self, tmp_path): @@ -227,11 +222,11 @@ def test_rejects_a_benchmark_without_the_requested_shared_prefix(self, tmp_path) created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - def test_rejects_benchmarks_at_other_concurrencies(self, tmp_path): + def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): run = get_running_service_run() with pytest.raises( - CLIError, match=r"concurrencies \[8\] do not match the requested \[4\]" + CLIError, match="concurrency of 8 does not match the requested concurrency of 4" ): build_verified_preset( run=run, @@ -244,44 +239,6 @@ def test_rejects_benchmarks_at_other_concurrencies(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - def test_rejects_a_partial_sweep(self, tmp_path): - run = get_running_service_run() - - with pytest.raises( - CLIError, match=r"concurrencies \[2, 4\] do not match the requested \[2, 4, 8\]" - ): - build_verified_preset( - run=run, - preset_configuration=_shared_prefix_configuration(concurrency="2..8"), - report=_sweep_report(run, [2, 4]), - workspace_path=tmp_path, - session_path=tmp_path, - preset_id="ab12cd34", - name=None, - created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), - ) - - def test_stores_every_point_and_selects_the_fastest_within_max_ttft(self, tmp_path): - run = get_running_service_run() - - preset = build_verified_preset( - run=run, - preset_configuration=_shared_prefix_configuration(concurrency="2..8", max_ttft=500), - # Reported out of order; throughput rises with concurrency, but the top - # point breaches the TTFT cap. - report=_sweep_report( - run, [8, 2, 4], duration_seconds={2: 8.0, 4: 4.0, 8: 2.0}, ttft_p50={8: 900.0} - ), - workspace_path=tmp_path, - session_path=tmp_path, - preset_id="ab12cd34", - name=None, - created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), - ) - - assert [b.workload.concurrency for b in preset.benchmarks] == [2, 4, 8] - assert preset.benchmark.workload.concurrency == 4 - def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): run = get_running_service_run() report = _dataset_report(run, dataset="spec_bench") @@ -389,22 +346,22 @@ def test_redacts_known_secret_in_benchmark_command_instead_of_failing(self, tmp_ run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmarks"][0]["command"] = ( + data["benchmark"]["command"] = ( "python bench.py --header 'Authorization: Bearer sk-live-0123456789abcdef'" ) report = self._load(tmp_path, data, redacted_values=("sk-live-0123456789abcdef",)) assert isinstance(report, PresetAgentSuccess) - assert report.benchmarks - assert report.benchmarks[0].command.endswith("Bearer [redacted]'") - assert "sk-live" not in report.benchmarks[0].command + assert report.benchmark is not None + assert report.benchmark.command.endswith("Bearer [redacted]'") + assert "sk-live" not in report.benchmark.command def test_still_rejects_unknown_bearer_token(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmarks"][0]["command"] = ( + data["benchmark"]["command"] = ( "curl -H 'Authorization: Bearer sk-unknown-9876543210fedcba'" ) @@ -417,7 +374,7 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmarks"][0]["command"] = ( + data["benchmark"]["command"] = ( "./benchenv/bin/python bench_service.py --base $DSTACK_SERVER_URL/x" " (auth via DSTACK_TOKEN bearer header from env)" ) @@ -425,46 +382,11 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): report = self._load(tmp_path, data, redacted_values=()) assert isinstance(report, PresetAgentSuccess) - assert report.benchmarks - assert "bearer header" in report.benchmarks[0].command - - def test_reads_a_pre_sweep_success_as_a_one_point_sweep(self, tmp_path): - # A session detached under a pre-sweep CLI finishes with that CLI's schema. - run = get_running_service_run() - data = get_successful_preset_report(run).model_dump() - data["run_id"] = str(data["run_id"]) - data["benchmark"] = data.pop("benchmarks")[0] - - report = self._load(tmp_path, data, redacted_values=()) - - assert isinstance(report, PresetAgentSuccess) - assert [b.workload.concurrency for b in report.benchmarks] == [1] - - def test_reads_a_sweep_success_unchanged(self, tmp_path): - run = get_running_service_run() - data = get_successful_preset_report(run).model_dump() - data["run_id"] = str(data["run_id"]) - second = {**data["benchmarks"][0], "workload": {**data["benchmarks"][0]["workload"]}} - second["workload"]["concurrency"] = 2 - data["benchmarks"].append(second) - - report = self._load(tmp_path, data, redacted_values=()) - - assert isinstance(report, PresetAgentSuccess) - assert [b.workload.concurrency for b in report.benchmarks] == [1, 2] - - def test_leaves_a_failure_report_alone(self, tmp_path): - report = self._load( - tmp_path, - {"success": False, "failure_summary": "no configuration met max_ttft"}, - redacted_values=(), - ) - - assert isinstance(report, PresetAgentFailure) - assert report.failure_summary == "no configuration met max_ttft" + assert report.benchmark is not None + assert "bearer header" in report.benchmark.command -def _shared_prefix_configuration(**overrides: Any) -> PresetConfiguration: +def _shared_prefix_configuration() -> PresetConfiguration: """The configuration from dstackai/dstack#4198: a shared prefix and no dataset.""" return PresetConfiguration.model_validate( { @@ -477,7 +399,6 @@ def _shared_prefix_configuration(**overrides: Any) -> PresetConfiguration: "input_tokens": 131072, "output_tokens": 512, "shared_prefix_tokens": 130048, - **overrides, } ) @@ -486,29 +407,6 @@ def _shared_prefix_report(run: Run, **workload: Any) -> PresetAgentSuccess: return _report_with_workload(run, {**SHARED_PREFIX_WORKLOAD, **workload}) -def _sweep_report( - run: Run, - concurrencies: list[int], - *, - duration_seconds: dict[int, float] | None = None, - ttft_p50: dict[int, float] | None = None, -) -> PresetAgentSuccess: - """One shared-prefix benchmark per concurrency, in the given order. The - metrics are the fixture's unless overridden per concurrency.""" - benchmarks = [] - for concurrency in concurrencies: - benchmark = get_preset_benchmark() - benchmark.workload = PresetWorkload.model_validate( - {**SHARED_PREFIX_WORKLOAD, "concurrency": concurrency} - ) - if duration_seconds and concurrency in duration_seconds: - benchmark.metrics.duration_seconds = duration_seconds[concurrency] - if ttft_p50 and concurrency in ttft_p50: - benchmark.metrics.ttft_ms.p50 = ttft_p50[concurrency] - benchmarks.append(benchmark) - return get_successful_preset_report(run).model_copy(update={"benchmarks": benchmarks}) - - def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: # A dataset defines the request shape, so the workload records what it measured. return _report_with_workload( @@ -527,4 +425,4 @@ def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: def _report_with_workload(run: Run, workload: dict[str, Any]) -> PresetAgentSuccess: benchmark = get_preset_benchmark() benchmark.workload = PresetWorkload.model_validate(workload) - return get_successful_preset_report(run).model_copy(update={"benchmarks": [benchmark]}) + return get_successful_preset_report(run).model_copy(update={"benchmark": benchmark}) diff --git a/src/tests/_internal/cli/services/presets/test_workspace.py b/src/tests/_internal/cli/services/presets/test_workspace.py index 3c60ab227a..a10832688e 100644 --- a/src/tests/_internal/cli/services/presets/test_workspace.py +++ b/src/tests/_internal/cli/services/presets/test_workspace.py @@ -1,5 +1,3 @@ -import json - import pytest from dstack._internal.cli.services.presets.session import PresetSession @@ -83,60 +81,3 @@ def test_a_session_without_records_warns(self, tmp_path, capsys): assert "empty000 has no records" in capsys.readouterr().err assert not (workspace.path / "previous" / "empty000").exists() - - def test_upgrades_pre_sweep_records_to_the_sweep_format(self, tmp_path): - session = _previous_session(tmp_path) - root = session.path - benchmark = {"workload": {"concurrency": 8}, "metrics": {}} - (root / "constraints.json").write_text(json.dumps({"max_ttft": 2000, "concurrency": 8})) - (root / "trials" / "1" / "trial.json").write_text( - json.dumps({"context_length": 8192, "benchmark": benchmark, "learned": "x"}) - ) - (root / "trials" / "2").mkdir() - # A trial that never produced a benchmark. - (root / "trials" / "2" / "trial.json").write_text( - json.dumps({"benchmark": None, "learned": "y", "failed": True}) - ) - (root / "final_report.json").write_text( - json.dumps({"success": True, "benchmark": benchmark}) - ) - workspace = _workspace(tmp_path) - - install_previous_records(workspace, [session]) - - target = workspace.path / "previous" / "8d3b01aa" - assert json.loads((target / "constraints.json").read_text()) == { - "max_ttft": 2000, - "concurrency": [8], - } - trial = json.loads((target / "trials" / "1" / "trial.json").read_text()) - # Same position as the field it replaces. - assert list(trial) == ["context_length", "benchmarks", "learned"] - assert trial["benchmarks"] == [benchmark] - failed = json.loads((target / "trials" / "2" / "trial.json").read_text()) - assert failed["benchmarks"] is None - report = json.loads((target / "final_report.json").read_text()) - assert report == {"success": True, "benchmarks": [benchmark]} - # The session's own records are not rewritten. - assert json.loads((root / "constraints.json").read_text())["concurrency"] == 8 - - def test_copies_sweep_records_unchanged(self, tmp_path): - session = _previous_session(tmp_path) - text = '{"benchmarks": [{"workload": {"concurrency": 4}}], "learned": "x"}' - (session.path / "trials" / "1" / "trial.json").write_text(text) - workspace = _workspace(tmp_path) - - install_previous_records(workspace, [session]) - - copied = workspace.path / "previous" / "8d3b01aa" / "trials" / "1" / "trial.json" - assert copied.read_text() == text - - def test_copies_an_unreadable_record_as_written(self, tmp_path): - session = _previous_session(tmp_path) - (session.path / "trials" / "1" / "trial.json").write_text('{"benchmark": {"met') - workspace = _workspace(tmp_path) - - install_previous_records(workspace, [session]) - - copied = workspace.path / "previous" / "8d3b01aa" / "trials" / "1" / "trial.json" - assert copied.read_text() == '{"benchmark": {"met' diff --git a/src/tests/_internal/core/models/test_configurations.py b/src/tests/_internal/core/models/test_configurations.py index ec9f35a65e..a77656c43d 100644 --- a/src/tests/_internal/core/models/test_configurations.py +++ b/src/tests/_internal/core/models/test_configurations.py @@ -1344,49 +1344,6 @@ def test_defaults_to_a_synthetic_workload(self): assert configuration.dataset is None -class TestPresetConfigurationConcurrency: - @pytest.mark.parametrize( - ("value", "expected"), - [ - pytest.param(10, [10], id="int"), - pytest.param("2,3", [2, 3], id="comma"), - pytest.param([3, 2, 2], [2, 3], id="list-sorted-deduped"), - pytest.param("2..16", [2, 4, 8, 16], id="range"), - pytest.param("8..8", [8], id="range-single"), - pytest.param(" 2 .. 8 ", [2, 4, 8], id="range-spaces"), - ], - ) - def test_normalizes_to_list(self, value, expected): - conf = PresetConfiguration.model_validate( - {"type": "preset", "base": "m", "concurrency": value} - ) - - assert conf.concurrency == expected - - @pytest.mark.parametrize( - ("value", "error"), - [ - pytest.param("3..16", "not reachable by doubling from 3", id="range-off-chain"), - pytest.param("..16", "needs both ends", id="range-no-start"), - pytest.param("2..", "needs both ends", id="range-no-end"), - pytest.param("16..2", "up to its end", id="range-descending"), - pytest.param([], "at least 1", id="empty-list"), - pytest.param(0, "greater than 0", id="zero"), - pytest.param("2,x", "valid integer", id="comma-garbage"), - ], - ) - def test_rejects(self, value, error): - with pytest.raises(ValidationError, match=error): - PresetConfiguration.model_validate( - {"type": "preset", "base": "m", "concurrency": value} - ) - - def test_stays_unset_when_omitted(self): - conf = PresetConfiguration.model_validate({"type": "preset", "base": "m"}) - - assert conf.concurrency is None - - class TestPresetConfigurationSchema: def test_schema_does_not_require_model(self): # `model` is filled from the `base`/`repo` shorthand by a before-validator, diff --git a/src/tests/_internal/core/models/test_presets.py b/src/tests/_internal/core/models/test_presets.py index bd1f8133d3..e12a881663 100644 --- a/src/tests/_internal/core/models/test_presets.py +++ b/src/tests/_internal/core/models/test_presets.py @@ -5,7 +5,6 @@ from dstack._internal.core.models.presets import ( PresetBenchmark, - select_benchmark, validate_preset_file_path, validate_preset_file_paths, ) @@ -153,32 +152,3 @@ def test_keeps_a_dataset_named_random_without_a_prefix(self): benchmark = PresetBenchmark.model_validate(data) assert benchmark.workload.dataset == "random" - - -def _benchmark(*, concurrency: int, tok_s: float, ttft_p50: float) -> PresetBenchmark: - data = get_benchmark_data(get_workload_data(concurrency=concurrency)) - # The fixture's 2048 output tokens over `duration` give `tok_s`. - data["metrics"]["duration_seconds"] = 2048 / tok_s - data["metrics"]["ttft_ms"]["p50"] = ttft_p50 - return PresetBenchmark.model_validate(data) - - -class TestSelectBenchmark: - def test_picks_the_fastest_point_within_max_ttft(self): - over = _benchmark(concurrency=16, tok_s=600, ttft_p50=900) - ok = _benchmark(concurrency=8, tok_s=500, ttft_p50=300) - - assert select_benchmark([ok, over], max_ttft=500) is ok - - def test_falls_back_to_the_closest_ttft_when_none_passes(self): - # Everything breached: the record is how near it got, not how fast it ran. - closer = _benchmark(concurrency=8, tok_s=500, ttft_p50=900) - faster = _benchmark(concurrency=16, tok_s=600, ttft_p50=950) - - assert select_benchmark([closer, faster], max_ttft=500) is closer - - def test_breaks_a_ttft_tie_by_throughput(self): - slow = _benchmark(concurrency=8, tok_s=500, ttft_p50=900) - fast = _benchmark(concurrency=16, tok_s=600, ttft_p50=900) - - assert select_benchmark([slow, fast], max_ttft=500) is fast From 89cecc75fb566685d403895fc66036d046255c0e Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Wed, 18 Nov 2026 11:30:31 +0545 Subject: [PATCH 07/12] Support concurrency sweep in presets --- src/dstack/_internal/cli/commands/preset.py | 9 +- .../_internal/cli/models/preset_agent.py | 52 +++- src/dstack/_internal/cli/models/presets.py | 37 ++- .../_internal/cli/services/presets/build.py | 8 +- .../_internal/cli/services/presets/output.py | 276 ++++++++++-------- .../presets/resources/system_prompt.md | 10 +- .../_internal/cli/services/presets/session.py | 234 ++++++++------- .../_internal/cli/services/presets/verify.py | 40 ++- .../_internal/core/models/configurations.py | 27 +- src/dstack/_internal/core/models/presets.py | 46 ++- 10 files changed, 471 insertions(+), 268 deletions(-) diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py index 28ba5e75ef..bae363f538 100644 --- a/src/dstack/_internal/cli/commands/preset.py +++ b/src/dstack/_internal/cli/commands/preset.py @@ -14,6 +14,7 @@ from dstack._internal.cli.models.presets import ( AnyStoredPreset, PresetListOutput, + PresetSessionEntry, UnverifiedPreset, ) from dstack._internal.cli.services.completion import ProjectNameCompleter @@ -284,7 +285,7 @@ def _list(self, args: argparse.Namespace) -> None: def _list_presets_and_sessions( self, *, base: str | None, repo: str | None - ) -> tuple[list[AnyStoredPreset], list[dict]]: + ) -> tuple[list[AnyStoredPreset], list[PresetSessionEntry]]: self._reconcile() presets = PresetStore().list() sessions = list_preset_sessions() @@ -552,15 +553,13 @@ def _filter_presets( def _session_matches_model( - session: dict, + session: PresetSessionEntry, *, base: str | None, repo: str | None, repo_to_base: dict[str, str], ) -> bool: - model = str(session.get("model") or "") - if not model: - return False + model = session.state.model if repo is not None: return model == repo return model == base or repo_to_base.get(model) == base diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 812ddce17e..1f55d2fa91 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -9,11 +9,13 @@ ValidationError, ValidatorFunctionWrapHandler, WrapValidator, + model_validator, ) +from typing_extensions import Self from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.configurations import PresetAgentProvider, ServiceConfiguration -from dstack._internal.core.models.presets import PresetBenchmark +from dstack._internal.core.models.presets import PresetBenchmark, PresetResult class PresetAgentInvalidService(CoreModel): @@ -52,8 +54,7 @@ class PresetAgentSuccess(CoreModel): trial: PositiveInt base: Annotated[str, Field(min_length=1)] model: Annotated[str, Field(min_length=1)] - context_length: PositiveInt - benchmark: PresetBenchmark + result: Annotated[list[PresetResult], Field(min_length=1)] class PresetAgentFailure(CoreModel): @@ -66,6 +67,51 @@ class PresetAgentFailure(CoreModel): AnyPresetAgentResult = Union[PresetAgentSuccess, PresetAgentFailure] +class PresetTrialBenchmark(PresetBenchmark): + """A trial's benchmark as the agent recorded it. The preset-quality rules of + `PresetBenchmark.validate_metrics` apply to the final report; a trial may + have failed requests, and the listing still shows what it measured.""" + + @model_validator(mode="after") + def validate_metrics(self) -> Self: + return self + + +class PresetTrialResult(CoreModel): + """One `result` entry of `trials//trial.json`, as the system prompt + defines it. `learned` is left out because the agent writes it for its own reading + in later sessions, and not used by the CLI""" + + concurrency: Optional[PositiveInt] = None + context_length: Optional[PositiveInt] = None + benchmark: Optional[PresetTrialBenchmark] = None + failed: bool = False + + +class PresetTrial(CoreModel): + """The `trials//trial.json` record as defined in the system_prompt.md.""" + + resources: Optional[dict[str, Any]] = None + groups: Optional[list[list[dict[str, Any]]]] = None + # Required and non-empty: `system_prompt.md` states that `trial.json` has + # one `result` entry per concurrency value. + result: Annotated[list[PresetTrialResult], Field(min_length=1)] + + @model_validator(mode="before") + @classmethod + def convert_to_result_list(cls, data: Any) -> Any: + if isinstance(data, dict) and "result" not in data and "benchmark" in data: + data = dict(data) + data["result"] = [ + { + key: data.pop(key) + for key in ("benchmark", "context_length", "failed") + if key in data + } + ] + return data + + PresetSessionStatus = Literal["running", "interrupted", "success", "failed"] # The session models carry no defaults: `session.json` is this CLI's own state diff --git a/src/dstack/_internal/cli/models/presets.py b/src/dstack/_internal/cli/models/presets.py index 75d469d51a..70b9d5c003 100644 --- a/src/dstack/_internal/cli/models/presets.py +++ b/src/dstack/_internal/cli/models/presets.py @@ -4,9 +4,14 @@ from pydantic import Field, PositiveInt from typing_extensions import Annotated +from dstack._internal.cli.models.preset_agent import ( + PresetSessionState, + PresetSessionStatus, + PresetTrial, +) from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.configurations import PresetConfiguration -from dstack._internal.core.models.presets import PortablePreset +from dstack._internal.core.models.presets import AnyPresetConstraints, PortablePreset class BasePreset(CoreModel): @@ -45,3 +50,33 @@ class PulledPreset(BasePreset, PortablePreset): class PresetListOutput(CoreModel): presets: List[AnyStoredPreset] + + +class PresetTrialsSummary(CoreModel): + """What the session row of `dstack preset list` shows about a session's + completed trials.""" + + count: int + # The trial with the highest tok/s that met the constraints. + best: Optional[PresetTrial] = None + # The trial with the highest tok/s but broke the constraints. + best_failed: Optional[PresetTrial] = None + # One value per trial in order, for the sparkline; None for a trial that + # produced no benchmark. + series: list[Optional[float]] + # Parallel to `series`: True where the trial broke the constraints. + failed: list[bool] + # The hardware of the shown trial, `best` or else `best_failed`; when neither + # exists, of the last trial that recorded it, so the session still shows what + # it ran on. + gpu: Optional[str] = None + + +class PresetSessionEntry(CoreModel): + id: str + state: PresetSessionState + name: Optional[str] + status: PresetSessionStatus + trials: PresetTrialsSummary + verification: Optional[str] + constraints: Optional[AnyPresetConstraints] diff --git a/src/dstack/_internal/cli/services/presets/build.py b/src/dstack/_internal/cli/services/presets/build.py index 1c3137f7aa..cf2d4b183e 100644 --- a/src/dstack/_internal/cli/services/presets/build.py +++ b/src/dstack/_internal/cli/services/presets/build.py @@ -9,7 +9,7 @@ from dstack._internal.core.models.instances import Resources from dstack._internal.core.models.presets import ( PRESET_EXCLUDED_FIELDS, - PresetBenchmark, + PresetResult, PresetVerificationReplicaGroup, ) from dstack._internal.core.models.resources import ( @@ -31,8 +31,7 @@ def build_preset( verification_replica_groups: list[PresetVerificationReplicaGroup], base_model: str, repo: str, - context_length: int, - benchmark: PresetBenchmark, + result: list[PresetResult], configuration: PresetConfiguration, best_trial: int, preset_id: str, @@ -48,12 +47,11 @@ def build_preset( base=base_model, id=preset_id, repo=repo, - context_length=context_length, best_trial=best_trial, configuration=configuration, created_at=created_at, service=service, - benchmark=benchmark, + result=result, verified_on=verification_replica_groups, ) diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 86f39d54c7..a5af1de5d7 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -1,13 +1,22 @@ from collections import defaultdict -from typing import Any, Optional, Sequence +from typing import Any, Callable, Iterable, Optional, Sequence, Union from rich.table import Table -from dstack._internal.cli.models.presets import AnyStoredPreset, VerifiedPreset +from dstack._internal.cli.models.preset_agent import PresetTrialResult +from dstack._internal.cli.models.presets import ( + AnyStoredPreset, + PresetSessionEntry, + VerifiedPreset, +) from dstack._internal.cli.utils.common import add_row_from_dict, console from dstack._internal.core.models.configurations import PresetConfiguration -from dstack._internal.core.models.presets import PresetWorkload -from dstack._internal.utils.common import parse_datetime, pretty_date, pretty_resources +from dstack._internal.core.models.presets import ( + PresetDatasetConstraints, + PresetResult, + PresetWorkload, +) +from dstack._internal.utils.common import pretty_date, pretty_resources _STATUS_DISPLAY = { "verified": ("verified", "secondary"), @@ -24,32 +33,27 @@ def _format_status(status: str) -> str: return f"[{style}]{text}[/]" if style else text -def _verifying(session: dict[str, Any]) -> bool: - return isinstance(session.get("verification"), dict) +def _verifying(session: PresetSessionEntry) -> bool: + return session.verification is not None _SPARK_BLOCKS = "▁▂▃▄▅▆▇█" -def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: - if not isinstance(session, dict): +def _format_trial_spark(session: PresetSessionEntry) -> str: + series = session.trials.series + failed = session.trials.failed + if not series: return "" - trials = session.get("trials") - series = trials.get("series") if isinstance(trials, dict) else None - if not isinstance(series, list) or not series: - return "" - failed = trials.get("failed") - if not isinstance(failed, list) or len(failed) != len(series): - failed = [False] * len(series) - values = [v for v in series if isinstance(v, (int, float))] + values = [v for v in series if v is not None] if not values: return "·" * len(series) high = max(values) - passed = [v for v, f in zip(series, failed) if isinstance(v, (int, float)) and not f] + passed = [v for v, f in zip(series, failed) if v is not None and not f] best = max(passed) if passed else max(values) out = [] for value, is_failed in zip(series, failed): - if not isinstance(value, (int, float)): + if value is None: out.append("[indian_red1]·[/]") continue glyph = ( @@ -65,25 +69,22 @@ def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: return "".join(out) -def _format_trial_progress(session: Optional[dict[str, Any]], *, in_flight: bool = False) -> str: - if not isinstance(session, dict): - return "" - trials = session.get("trials") - trials_num = session.get("trials_num") - if not isinstance(trials, dict) or not (trials.get("count") or isinstance(trials_num, int)): +def _format_trial_progress(session: PresetSessionEntry, *, in_flight: bool = False) -> str: + count = session.trials.count + trials_num = session.state.trials_num + if not count and trials_num is None: return "" - count = trials.get("count") or 0 if in_flight: - count = min(count + 1, trials_num) if isinstance(trials_num, int) else count + 1 + count = min(count + 1, trials_num) if trials_num is not None else count + 1 progress = str(count) - if isinstance(trials_num, int): + if trials_num is not None: progress += f"/{trials_num}" return f" [secondary]({progress})[/]" def print_presets( presets: Sequence[AnyStoredPreset], - sessions: Optional[list[dict[str, Any]]] = None, + sessions: Optional[list[PresetSessionEntry]] = None, verbose: bool = False, all_presets: bool = False, limit: Optional[int] = None, @@ -98,7 +99,7 @@ def print_presets( def get_presets_table( presets: Sequence[AnyStoredPreset], - sessions: Optional[list[dict[str, Any]]] = None, + sessions: Optional[list[PresetSessionEntry]] = None, verbose: bool = False, all_presets: bool = False, limit: Optional[int] = None, @@ -120,16 +121,15 @@ def get_presets_table( for preset in presets: presets_by_base[preset.base].append(preset) repo_to_base[preset.repo] = preset.base - sessions_by_model: dict[str, list[dict[str, Any]]] = defaultdict(list) - creations_by_id: dict[str, dict[str, Any]] = {} + sessions_by_model: dict[str, list[PresetSessionEntry]] = defaultdict(list) + creations_by_id: dict[str, PresetSessionEntry] = {} for session in sessions or []: - session_id = str(session.get("id")) - if session_id in preset_ids: + if session.id in preset_ids: # The stored artifact is the completion record. Cleanup can be # interrupted before session.json advances to `success`. - creations_by_id[session_id] = session + creations_by_id[session.id] = session continue - model = str(session.get("model") or "unknown") + model = session.state.model sessions_by_model[repo_to_base.get(model, model)].append(session) # Same contract as `dstack ps`: one flat list, newest first (base is a column, @@ -139,12 +139,12 @@ def get_presets_table( rows += [(preset.created_at.isoformat(), preset, True) for preset in preset_list] for session_list in sessions_by_model.values(): rows += [ - (str(session.get("created_at") or ""), session, False) for session in session_list + (session.state.created_at.isoformat(), session, False) for session in session_list ] rows.sort(key=lambda r: r[0], reverse=True) only_active = not all_presets and limit is None if only_active: - active = [r for r in rows if not r[2] and str(r[1].get("status")) == "running"] + active = [r for r in rows if not r[2] and r[1].status == "running"] rows = active or rows[:1] for _, item, is_preset in rows[:limit] if limit is not None else rows: @@ -155,93 +155,62 @@ def get_presets_table( return table -def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False) -> None: - created = "" - created_at = session.get("created_at") - if isinstance(created_at, str): - try: - created = pretty_date(parse_datetime(created_at)) - except ValueError: - created = created_at - benchmark = "" - gpu = "" - status_key = str(session.get("status", "")) +def _add_session(table: Table, session: PresetSessionEntry, *, verbose: bool = False) -> None: + status_key = session.status if status_key == "running" and _verifying(session): status_key = "verifying" status = _format_status(status_key) + _format_trial_progress( session, in_flight=status_key == "running" ) - trials = session.get("trials") - best = trials.get("best") if isinstance(trials, dict) else None + trials = session.trials # Nothing passed: fall back to the fastest attempt that did not. - breached = not isinstance(best, dict) - if breached and isinstance(trials, dict): - best = trials.get("best_failed") - if isinstance(best, dict): - gpu = best.get("gpu") or "" - if not gpu and isinstance(trials, dict): - # Trials can record hardware without ever producing a benchmark. - gpu = trials.get("gpu") or "" - constraints = session.get("constraints") or {} - parts = [] + breached = trials.best is None + best = trials.best or trials.best_failed + constraints = session.constraints objective = [] - if dataset := constraints.get("dataset"): - objective.append(f"data={dataset}") - if constraints.get("input_tokens") and constraints.get("output_tokens"): + if isinstance(constraints, PresetDatasetConstraints): + objective.append(f"data={constraints.dataset}") + elif constraints is not None: objective.append( - f"io={_format_token_count(constraints['input_tokens'])}" - f"/{_format_token_count(constraints['output_tokens'])}" + f"io={_format_token_count(constraints.input_tokens)}" + f"/{_format_token_count(constraints.output_tokens)}" ) - input_tokens = constraints.get("input_tokens") - if input_tokens: - shared_prefix_tokens = constraints.get("shared_prefix_tokens") or 0 - share = round(100 * shared_prefix_tokens / input_tokens) + share = round(100 * constraints.shared_prefix_tokens / constraints.input_tokens) if share: objective.append(f"prefix={share}%") - concurrency = (best or {}).get("concurrency") or constraints.get("concurrency") + # The shown trial's sweep when it has one, else what was requested. + concurrency = _result_concurrencies(best.result) if best is not None else [] + if not concurrency and constraints is not None: + concurrency = constraints.concurrency if concurrency: - objective.append(f"c={concurrency}") - min_context_length = constraints.get("min_context_length") - if verbose and isinstance(min_context_length, int): - objective.append(f"ctx>={_format_token_count(min_context_length)}") - max_ttft = constraints.get("max_ttft") - if verbose and isinstance(max_ttft, (int, float)): - objective.append(f"ttft<={_format_duration_ms(max_ttft)}") - if isinstance(best, dict): - tps = _format_number(best["tok_s"]) + objective.append(f"c={_format_concurrency(concurrency)}") + if verbose and constraints is not None: + objective.append(f"ctx>={_format_token_count(constraints.min_context_length)}") + objective.append(f"ttft<={_format_duration_ms(constraints.max_ttft)}") + benchmark = "" + if best is not None: if objective: - # Lead with per-user tps: comparable across rows regardless of concurrency. - tpot_ms = best.get("tpot_ms") - if isinstance(tpot_ms, (int, float)) and tpot_ms > 0: - parts.append(f"tps/user={_format_number(1000 / tpot_ms)}") - if verbose: - parts.append(f"tps={tps}") - ttft_ms = best.get("ttft_ms") - if isinstance(ttft_ms, (int, float)): - parts.append(f"ttft={_format_duration_ms(ttft_ms)}") - context_length = best.get("context_length") - if isinstance(context_length, int): - parts.append(f"ctx={_format_token_count(context_length)}") - benchmark = " ".join(parts) + benchmark = _format_result_benchmark(best.result, verbose=verbose) else: - benchmark = f"c={best.get('concurrency')} tps={tps}" - benchmark = benchmark.strip() + benchmarks = [r.benchmark for r in best.result if r.benchmark is not None] + tps = _format_range([b.effective_output_tok_per_s for b in benchmarks], _format_number) + benchmark = f"c={_format_concurrency(concurrency)} tps={tps}" if benchmark and breached: # Marked, not only dimmed: colour alone is not a signal. benchmark = f"[secondary]*{benchmark}[/]" add_row_from_dict( table, { - "ID": str(session.get("id", "")), - "BASE": str(session.get("model") or ""), - "NAME": str(session.get("name") or ""), - "GPU": gpu, - "RESOURCES": gpu, + "ID": session.id, + "BASE": session.state.model, + "NAME": session.name or "", + "GPU": trials.gpu or "", + "RESOURCES": trials.gpu or "", "": _format_trial_spark(session), "CONSTRAINTS": f"[secondary]{' '.join(objective)}[/]" if objective else "", "BENCHMARK": benchmark, "STATUS": status, - "SUBMITTED": created, + "SUBMITTED": pretty_date(session.state.created_at), }, ) @@ -251,7 +220,7 @@ def _add_preset( preset: AnyStoredPreset, *, verbose: bool, - creation: Optional[dict[str, Any]] = None, + creation: Optional[PresetSessionEntry] = None, ) -> None: groups = preset.service.replica_groups row = { @@ -259,8 +228,9 @@ def _add_preset( "NAME": preset.name or "", "RESOURCES": _format_resources(groups[0].resources, verbose=verbose), "BASE": preset.base, - "STATUS": _format_status(preset.status) + _format_trial_progress(creation), - "": _format_trial_spark(creation), + "STATUS": _format_status(preset.status) + + (_format_trial_progress(creation) if creation is not None else ""), + "": _format_trial_spark(creation) if creation is not None else "", "CONSTRAINTS": format_preset_objective( preset, verbose=verbose, @@ -288,7 +258,7 @@ def format_preset_objective( *, verbose: bool = False, ) -> str: - workload = preset.benchmark.workload + workload = _measured_workload_from_preset(preset) if isinstance(preset, VerifiedPreset): return _format_creation_objective(preset.configuration, workload, verbose=verbose) # A pulled preset carries no creation context; the measured workload is its @@ -304,10 +274,25 @@ def format_preset_objective( share = round(100 * workload.shared_prefix_tokens / workload.input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={workload.concurrency}") + parts.append(f"c={_format_concurrency([r.concurrency for r in preset.result])}") return f"[secondary]{' '.join(parts)}[/]" +def _measured_workload_from_preset(preset: AnyStoredPreset) -> PresetWorkload: + for r in preset.result: + if r.benchmark is not None: + return r.benchmark.workload + # Unreachable for a stored preset: verification requires at least one + # completed benchmark. Kept so the function returns on every path. + raise ValueError(f"preset {preset.id} has no measured workload") + + +def _format_concurrency(values: list[int]) -> str: + if len(values) == 1: + return str(values[0]) + return f"{min(values)}..{max(values)}" + + def _format_creation_objective( configuration: PresetConfiguration, workload: PresetWorkload, *, verbose: bool ) -> str: @@ -323,7 +308,7 @@ def _format_creation_objective( share = round(100 * (configuration.shared_prefix_tokens or 0) / input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={configuration.concurrency or workload.concurrency}") + parts.append(f"c={_format_concurrency(configuration.concurrency or [workload.concurrency])}") if verbose and configuration.min_context_length is not None: parts.append(f"ctx>={_format_token_count(configuration.min_context_length)}") if verbose and configuration.max_ttft is not None: @@ -336,32 +321,75 @@ def _breaches_constraints(preset: AnyStoredPreset) -> bool: # A pulled preset carries no requested constraints to breach. return False configuration = preset.configuration - metrics = preset.benchmark.metrics - if configuration.max_ttft is not None and metrics.ttft_ms.p50 > configuration.max_ttft: - return True - return configuration.min_context_length is not None and ( - preset.context_length < configuration.min_context_length - ) + # One breaching benchmark is enough: a service that misses its target at any + # measured concurrency is flagged. Entries without a benchmark are skipped. + for r in preset.result: + if r.benchmark is None: + continue + if ( + configuration.max_ttft is not None + and r.benchmark.metrics.ttft_ms.p50 > configuration.max_ttft + ): + return True + if ( + configuration.min_context_length is not None + and r.context_length is not None + and r.context_length < configuration.min_context_length + ): + return True + return False def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) -> str: - benchmark = preset.benchmark - metrics = benchmark.metrics - parts = [ - f"tps/user={_format_number(benchmark.effective_per_user_tok_per_s)}", - ] - if verbose: - parts.append(f"tps={_format_number(benchmark.effective_output_tok_per_s)}") - parts += [ - f"ttft={_format_duration_ms(metrics.ttft_ms.p50)}", - f"ctx={_format_token_count(preset.context_length)}", - ] - text = " ".join(parts) + text = _format_result_benchmark(preset.result, verbose=verbose) if _breaches_constraints(preset): return f"[secondary]*{text}[/]" return text +# Entries of a stored preset and of a session's trial share the fields read here. +AnyResult = Union[PresetResult, PresetTrialResult] + + +def _format_result_benchmark(result: Sequence[AnyResult], *, verbose: bool) -> str: + """Each metric as its range over the entries that have a benchmark.""" + benchmarks = [r.benchmark for r in result if r.benchmark is not None] + context_lengths = [r.context_length for r in result if r.context_length is not None] + # Lead with per-user tps: comparable across rows regardless of concurrency. + parts = [ + f"tps/user={_format_range([b.effective_per_user_tok_per_s for b in benchmarks], _format_number)}" + ] + if verbose: + parts.append( + f"tps={_format_range([b.effective_output_tok_per_s for b in benchmarks], _format_number)}" + ) + parts.append( + f"ttft={_format_range([b.metrics.ttft_ms.p50 for b in benchmarks], _format_duration_ms)}" + ) + if context_lengths: + parts.append(f"ctx={_format_range(context_lengths, _format_token_count)}") + return " ".join(parts) + + +def _result_concurrencies(result: Sequence[AnyResult]) -> list[int]: + """Old `trial.json` entries carry the concurrency only inside the benchmark.""" + values = [] + for r in result: + if r.concurrency is not None: + values.append(r.concurrency) + elif r.benchmark is not None: + values.append(r.benchmark.workload.concurrency) + return values + + +def _format_range(values: Iterable[float], format_value: Callable[[Any], str]) -> str: + values = list(values) + low, high = min(values), max(values) + if low == high: + return format_value(low) + return f"{format_value(low)}..{format_value(high)}" + + def _format_duration_ms(value: float) -> str: # 999.6 rounds to 1000, which must read as 1s rather than 1000ms. if value < 999.5: diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index d0dc937cae..6eb660e7c2 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -590,10 +590,7 @@ In case the service is using a router (see `## Cluster Placement`), run the final benchmark inside the router replica, directly against the router engine. Attach to it via `dstack attach --replica --job `. -If the service or its benchmark cannot be completed, stop that service, -pick the next-best trial, and repeat, until a service is verified or there -are no unverified trials left. Report the result accordingly (see -`# Final Report`). +If the service cannot be completed or none of its benchmarks at the concurrency values from `constraints.json` can be completed, stop that service, pick the next-best trial, and repeat, until a service is verified or there are no unverified trials left. Report the result accordingly (see `# Final Report`). When you report success, leave the final `dstack` service running: the caller verifies the live run and stops it afterwards. @@ -628,9 +625,10 @@ trial remained), include exactly: - `result`: list of final service result objects, one object for each value of `concurrency` from `constraints.json`. Each object has fields no other than below: - `concurrency`: the concurrency value for this result - `context_length`: the largest context verified for the final service at - this concurrency, as described in `## Benchmark` + this concurrency, as described in `## Benchmark`. `null` only when the benchmark couldn't be done at this concurrency. - `benchmark`: the final service benchmark at this concurrency, as - described in `## Benchmark` + described in `## Benchmark`. `null` only when the benchmark couldn't be done at this + concurrency. Set `final_report.json.base` as follows: diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index 9ce77213e3..ced935329c 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -25,12 +25,16 @@ PresetSessionState, PresetSessionStatus, PresetSessionWorkspace, + PresetTrial, + PresetTrialResult, ) +from dstack._internal.cli.models.presets import PresetSessionEntry, PresetTrialsSummary from dstack._internal.cli.utils.common import console, error_console from dstack._internal.compat import IS_WINDOWS from dstack._internal.core.errors import CLIError from dstack._internal.core.models.common import validate_extra_ignore, validate_json_extra_ignore from dstack._internal.core.models.configurations import PresetAgentProvider, PresetConfiguration +from dstack._internal.core.models.presets import AnyPresetConstraints from dstack._internal.utils.common import get_dstack_dir _PROGRESS_FILENAME = "progress.jsonl" @@ -515,7 +519,7 @@ def resolve_session_ref(ref: str) -> str: return ref -def list_preset_sessions() -> list[dict[str, Any]]: +def list_preset_sessions() -> list[PresetSessionEntry]: entries = [] for session in iter_preset_sessions(): path = session.path @@ -525,18 +529,21 @@ def list_preset_sessions() -> list[dict[str, Any]]: status = state.status if status == "running" and not session_process_alive(state): status = "interrupted" - entry = state.model_dump(mode="json") - entry["id"] = path.name - entry["name"] = claimed_session_name(state) - entry["status"] = status - entry["trials"] = _summarize_session_trials(path / TRIALS_DIRNAME) - entry["verification"] = _read_last_session_verification(path / SERVICE_DIRNAME) - entry["constraints"] = _read_session_constraints(path) - entries.append(entry) + entries.append( + PresetSessionEntry( + state=state, + id=path.name, + name=claimed_session_name(state), + status=status, + trials=_summarize_session_trials(path / TRIALS_DIRNAME), + verification=_read_last_session_verification(path / SERVICE_DIRNAME), + constraints=_read_session_constraints(path), + ) + ) return entries -def _read_session_constraints(path: Path) -> dict[str, Any]: +def _read_session_constraints(path: Path) -> Optional[AnyPresetConstraints]: """Reads the session's own copy first: it outlives the agent workspace (removed once the session finishes). The workspace copy is a backward-compat fallback for sessions recorded before the session-level copy existed.""" @@ -545,12 +552,12 @@ def _read_session_constraints(path: Path) -> dict[str, Any]: path / "workspace" / "w" / _CONSTRAINTS_FILENAME, ): try: - data = json.loads(candidate.read_text(encoding="utf-8")) - except (OSError, ValueError): + return validate_json_extra_ignore( + AnyPresetConstraints, candidate.read_text(encoding="utf-8") + ) + except (OSError, ValueError, ValidationError): continue - if isinstance(data, dict): - return data - return {} + return None def _numbered_subdirs(path: Path) -> list[Path]: @@ -571,101 +578,112 @@ def _read_record(path: Path) -> Optional[dict[str, Any]]: return record if isinstance(record, dict) else None -def _read_last_session_verification(path: Path) -> Optional[dict[str, Any]]: - """An attempt whose result file has not appeared yet is still in flight - (reported as verifying).""" +def _read_last_session_verification(path: Path) -> Optional[str]: + """The last attempt's status; an attempt whose result file has not appeared + yet is still in flight (reported as verifying).""" attempts = _numbered_subdirs(path) if not attempts: return None - last = attempts[-1] - record = _read_record(last / VERIFICATION_RESULT_FILENAME) + record = _read_record(attempts[-1] / VERIFICATION_RESULT_FILENAME) if record is not None and isinstance(record.get("status"), str): - return record - return {"status": "verifying"} + return record["status"] + return "verifying" -def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: - # TODO: Refactor this crap - must be explicit what is this and where is it used; also dicts are prohibited in dstack repo +def _summarize_session_trials(path: Path) -> PresetTrialsSummary: """A trial directory without `trial.json` is still in flight and is not counted.""" - records = [] + trials = [] for trial_dir in _numbered_subdirs(path): - record = _read_record(trial_dir / TRIAL_RESULT_FILENAME) - if record is not None: - records.append(record) - count = 0 - best: Optional[dict[str, Any]] = None - # The fastest trial that broke a constraint, shown only when nothing passed. - best_failed: Optional[dict[str, Any]] = None - # One entry per trial in order, `None` for a trial that produced no benchmark. - series: list[Optional[float]] = [] - # Parallel to `series`: True where the trial broke a constraint. - failed: list[bool] = [] - # Kept outside `best` so a run where nothing passed still shows what it ran on. + trial = _read_trial(trial_dir / TRIAL_RESULT_FILENAME) + if trial is not None: + trials.append(trial) + # Select the trial with the highest tok/s that meets the constraints, and + # show its sweep. When no trial meets them, fallback to the fastest + # failed one, marked in the listing. + best, best_failed = _select_best_and_best_failed(trials) + shown = best or best_failed + gpu = _format_trial_gpu(shown) if shown is not None else None + if gpu is None: + gpu = _last_trial_gpu(trials) + return PresetTrialsSummary( + count=len(trials), + best=best, + best_failed=best_failed, + series=[_max_tok_s(trial) for trial in trials], + failed=[_trial_failed(trial) for trial in trials], + gpu=gpu, + ) + + +def _last_trial_gpu(trials: list[PresetTrial]) -> Optional[str]: + """Trials can record hardware without ever producing a benchmark.""" gpu: Optional[str] = None - for record in records: - count += 1 - benchmark = record.get("benchmark") - failed.append(bool(record.get("failed"))) - record_gpu = _format_trial_gpu(record) - if record_gpu: - gpu = record_gpu - if not isinstance(benchmark, dict): - series.append(None) - continue - metrics = benchmark.get("metrics") or {} - workload = benchmark.get("workload") or {} - duration = metrics.get("duration_seconds") - tokens = metrics.get("total_output_tokens") - if not isinstance(duration, (int, float)) or duration <= 0: - series.append(None) - continue - if not isinstance(tokens, (int, float)): - series.append(None) - continue - tok_s = tokens / duration - series.append(tok_s) - # A failed trial keeps its benchmark — it is what the next trial learns - # from — but it is not a candidate for best, and promoting one would put - # a configuration that broke a constraint at the top of the listing. - if record.get("failed"): - if best_failed is None or tok_s > best_failed["tok_s"]: - best_failed = _trial_entry(tok_s, record, metrics, workload, record_gpu) + for trial in trials: + gpu = _format_trial_gpu(trial) or gpu + return gpu + + +def _read_trial(path: Path) -> Optional[PresetTrial]: + """None if `trial.json` is missing, half-written, or not a trial record.""" + record = _read_record(path) + if record is None: + return None + try: + return validate_extra_ignore(PresetTrial, record) + except ValidationError: + return None + + +def _trial_failed(trial: PresetTrial) -> bool: + """A trial failed when it broke the constraints at every concurrency.""" + return all(entry.failed for entry in trial.result) + + +def _passing_trial_results(trial: PresetTrial) -> list[PresetTrialResult]: + """The `result` where the trial met the constraints.""" + return [entry for entry in trial.result if not entry.failed] + + +def _max_tok_s(trial: PresetTrial) -> Optional[float]: + """The trial's score for selection and its sparkline bar: the highest tok/s + over its entries that have a benchmark; None when no entry has one.""" + values = [ + entry.benchmark.effective_output_tok_per_s + for entry in trial.result + if entry.benchmark is not None + ] + if not values: + return None + return max(values) + + +def _select_best_and_best_failed( + trials: list[PresetTrial], +) -> tuple[Optional[PresetTrial], Optional[PresetTrial]]: + """Selects the trial with the highest tok/s that meets the constraints, and + the trail with highest tok/s but failed to meet the constraints.""" + best: Optional[PresetTrial] = None + best_tok_s = 0.0 + best_failed: Optional[PresetTrial] = None + best_failed_tok_s = 0.0 + for trial in trials: + if _trial_failed(trial): + kept = trial + else: + kept = trial.model_copy(update={"result": _passing_trial_results(trial)}) + tok_s = _max_tok_s(kept) + if tok_s is None: continue - if best is None or tok_s > best["tok_s"]: - best = _trial_entry(tok_s, record, metrics, workload, record_gpu) - return { - "count": count, - "best": best, - "best_failed": best_failed, - "series": series, - "failed": failed, - "gpu": gpu, - } - - -def _trial_entry( - tok_s: float, - record: dict[str, Any], - metrics: dict[str, Any], - workload: dict[str, Any], - gpu: Optional[str], -) -> dict[str, Any]: - ttft = (metrics.get("ttft_ms") or {}).get("p50") - # Mean, not p50: MTP skews TPOT right, and mean is the delivered decode rate. - tpot = (metrics.get("tpot_ms") or {}).get("mean") - context_length = record.get("context_length") - return { - "tok_s": tok_s, - "tpot_ms": tpot if isinstance(tpot, (int, float)) and tpot > 0 else None, - "ttft_ms": ttft if isinstance(ttft, (int, float)) else None, - "context_length": context_length if isinstance(context_length, int) else None, - "concurrency": workload.get("concurrency"), - "gpu": gpu, - } - - -def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: + if _trial_failed(trial): + if best_failed is None or tok_s > best_failed_tok_s: + best_failed, best_failed_tok_s = kept, tok_s + elif best is None or tok_s > best_tok_s: + best, best_tok_s = kept, tok_s + return best, best_failed + + +def _format_trial_gpu(trial: PresetTrial) -> Optional[str]: """A trial records its hardware in one of two formats, as described in `system_prompt.md`: `{"resources": {...}}` without node groups, and `{"groups": [[...], ...]}` with them. @@ -679,7 +697,7 @@ def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: `dstack preset list -v`. """ counts: dict[str, int] = {} - for node in _trial_nodes(record): + for node in _trial_nodes(trial): spec = _format_gpu(node.get("gpu")) if spec: # Insertion order is group order, so the roles read in the order they ran. @@ -689,18 +707,10 @@ def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: return " + ".join(spec if n == 1 else f"{spec} x{n}" for spec, n in counts.items()) -def _trial_nodes(record: dict[str, Any]) -> list[dict[str, Any]]: - groups = record.get("groups") - if isinstance(groups, list): - return [ - node - for group in groups - if isinstance(group, list) - for node in group - if isinstance(node, dict) - ] - resources = record.get("resources") - return [resources] if isinstance(resources, dict) else [] +def _trial_nodes(trial: PresetTrial) -> list[dict[str, Any]]: + if trial.groups is not None: + return [node for group in trial.groups for node in group] + return [trial.resources] if trial.resources is not None else [] def _format_gpu(gpu: Any) -> Optional[str]: diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 643522c503..17eaeff205 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -29,6 +29,7 @@ from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.presets import ( PresetBenchmark, + PresetResult, PresetVerificationReplicaGroup, PresetWorkload, ) @@ -113,8 +114,7 @@ def build_verified_preset( # prompt has always used; the preset document calls it `repo`, so that # it cannot be confused with the service's client-facing model name. repo=report.model, - context_length=report.context_length, - benchmark=_normalized_benchmark(report.benchmark, preset_configuration), + result=_normalized_result(report.result, preset_configuration), best_trial=report.trial, configuration=preset_configuration, preset_id=preset_id, @@ -122,6 +122,17 @@ def build_verified_preset( ) +def _normalized_result( + result: list[PresetResult], configuration: PresetConfiguration +) -> list[PresetResult]: + return [ + r.model_copy(update={"benchmark": _normalized_benchmark(r.benchmark, configuration)}) + if r.benchmark is not None + else r + for r in result + ] + + def _verified_run_service(run: Run, report: PresetAgentSuccess) -> ServiceConfiguration: """The service the server actually runs, after proving the report talks about this run and the run is a live model service.""" @@ -141,7 +152,7 @@ def _check_report_answers_request( """The report must answer what the configuration asked: the same benchmark workload, and the requested model — exactly when it was exact, any variant of the base otherwise.""" - _check_workload_answers_request(report.benchmark.workload, configuration) + _check_result_answers_request(report.result, configuration) if configuration.model.allows_variant_selection: if report.base != configuration.model.api_model_name: raise CLIError("The agent's final report base does not match the requested model") @@ -149,6 +160,24 @@ def _check_report_answers_request( raise CLIError("The agent changed an exact model request") +def _check_result_answers_request( + result: list[PresetResult], configuration: PresetConfiguration +) -> None: + if all(r.benchmark is None for r in result): + raise CLIError("The agent's final report has no completed benchmark") + for r in result: + # A `null` benchmark is an entry the agent could not measure; a partial + # sweep is allowed, so there is nothing to compare for it. + if r.benchmark is None: + continue + if r.benchmark.workload.concurrency != r.concurrency: + raise CLIError( + f"The agent's final benchmark concurrency {r.concurrency} does not" + f" match the benchmark workload concurrency {r.benchmark.workload.concurrency}" + ) + _check_workload_answers_request(r.benchmark.workload, configuration) + + def _check_workload_answers_request( workload: PresetWorkload, configuration: PresetConfiguration ) -> None: @@ -172,7 +201,10 @@ def _check_workload_answers_request( f"The agent's final benchmark shared prefix of {workload.shared_prefix_tokens}" f" tokens does not match the requested {shared_prefix_tokens}" ) - if configuration.concurrency is not None and workload.concurrency != configuration.concurrency: + if ( + configuration.concurrency is not None + and workload.concurrency not in configuration.concurrency + ): raise CLIError( f"The agent's final benchmark concurrency of {workload.concurrency} does not match the" f" requested concurrency of {configuration.concurrency}" diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 6cf8e423ff..25b7e4617a 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -1774,12 +1774,13 @@ class PresetConfiguration( ), ] = None concurrency: Annotated[ - Optional[PositiveInt], + Optional[list[PositiveInt]], Field( + min_length=1, description=( - "The number of simultaneous requests used for benchmarks during preset" + "List of simultaneous request counts used for benchmarks during preset" " creation. Required for creation" - ) + ), ), ] = None input_tokens: Annotated[ @@ -1945,6 +1946,26 @@ def validate_prompt(cls, value: Any) -> Any: raise ValueError(f"Prompt must be at most {MAX_PROMPT_LENGTH} characters") return value + @field_validator( + "concurrency", mode="before", json_schema_input_type=Union[PositiveInt, list[PositiveInt]] + ) + @classmethod + def convert_single_concurrency_to_list(cls, value: Any) -> Any: + if isinstance(value, bool): + raise ValueError( + "`concurrency` must be a positive integer or a list of positive integers" + ) + if isinstance(value, int): + return [value] + return value + + @field_validator("concurrency") + @classmethod + def sort_concurrency(cls, value: Optional[list[PositiveInt]]) -> Optional[list[PositiveInt]]: + if value is None: + return None + return sorted(set(value)) + def _validate_model(value: Any, *, field: str) -> str: if not isinstance(value, str) or not value.strip(): diff --git a/src/dstack/_internal/core/models/presets.py b/src/dstack/_internal/core/models/presets.py index 95e87fb29f..7cc40f1081 100644 --- a/src/dstack/_internal/core/models/presets.py +++ b/src/dstack/_internal/core/models/presets.py @@ -1,5 +1,5 @@ import re -from typing import Any, List, Literal, Optional, Sequence +from typing import Any, List, Literal, Optional, Sequence, Union from pydantic import Field, PositiveFloat, PositiveInt, field_validator, model_validator from typing_extensions import Annotated, Self @@ -128,6 +128,15 @@ def validate_metrics(self) -> Self: return self +class PresetResult(CoreModel): + concurrency: PositiveInt + # The largest context the service was verified to serve at this concurrency. + # A preset needs a completed benchmark at only one of the concurrencies from constraints. + # This is why context_length and benchmark are optional. + context_length: Optional[PositiveInt] = None + benchmark: Optional[PresetBenchmark] = None + + class PresetVerificationReplicaGroup(CoreModel): # The service replica group this was measured for. name: str @@ -144,8 +153,6 @@ class PortablePreset(CoreModel): # The exact repo/path the service loads, which `base` is a variant of. Not # the client-facing API model name — that is `service.model`. repo: Annotated[str, Field(min_length=1)] - # The largest context the service was verified to serve. - context_length: PositiveInt # The verified run's spec configuration. The validator below keeps `name`, # `gateway`, and profile params unset (the deployer's choices) and requires # `model` and resources. Env keys the user declared as passthroughs hold @@ -153,11 +160,26 @@ class PortablePreset(CoreModel): # stored as-is. `files` paths are stored relative to the preset directory, # absolute after load. service: ServiceConfiguration - benchmark: PresetBenchmark + result: Annotated[List[PresetResult], Field(min_length=1)] # The hardware it was verified on: the actual resources of every running # replica, by service replica group. verified_on: List[PresetVerificationReplicaGroup] + @model_validator(mode="before") + @classmethod + def convert_to_result_list(cls, data: Any) -> Any: + if isinstance(data, dict) and "result" not in data and "benchmark" in data: + data = dict(data) + benchmark = data.pop("benchmark") + data["result"] = [ + { + "concurrency": benchmark["workload"]["concurrency"], + "context_length": data.pop("context_length", None), + "benchmark": benchmark, + } + ] + return data + @model_validator(mode="after") def validate_artifact(self) -> Self: service = self.service @@ -304,11 +326,20 @@ class PresetConstraints(CoreModel): min_context_length: PositiveInt max_ttft: PositiveInt trials_num: PositiveInt - concurrency: PositiveInt + concurrency: Annotated[list[PositiveInt], Field(min_length=1)] baseline: bool fleets: Annotated[list[str], Field(min_length=1)] env: list[str] + @field_validator("concurrency", mode="before") + @classmethod + def convert_single_concurrency_to_list(cls, value: Any) -> Any: + # `constraints.json` of a session created before the concurrency sweep + # holds a single value. + if isinstance(value, int) and not isinstance(value, bool): + return [value] + return value + class PresetRandomConstraints(PresetConstraints): """Constraints for the synthetic `random` dataset, which the request shape defines.""" @@ -324,6 +355,11 @@ class PresetDatasetConstraints(PresetConstraints): dataset: str +# `constraints.json` is one of the two; each has a required field the other lacks, +# so a parse picks the right one. +AnyPresetConstraints = Union[PresetDatasetConstraints, PresetRandomConstraints] + + def _validate_model(value: Any, *, field: str) -> str: if not isinstance(value, str) or not value.strip(): raise ValueError(f"Preset model {field} must be a non-empty string") From 421a5d7053428fe133031d9b60a0b26e5401952e Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Wed, 18 Nov 2026 13:41:41 +0545 Subject: [PATCH 08/12] Update Old Preset Tests --- .../_internal/cli/models/preset_agent.py | 18 +- src/dstack/_internal/cli/models/presets.py | 5 +- .../_internal/cli/commands/test_preset.py | 8 +- src/tests/_internal/cli/common.py | 105 +++++- .../cli/services/presets/test_agent.py | 93 ++--- .../cli/services/presets/test_create.py | 10 +- .../cli/services/presets/test_output.py | 347 ++++++++---------- .../services/presets/test_report_schema.py | 64 ++-- .../cli/services/presets/test_store.py | 23 +- .../cli/services/presets/test_verify.py | 56 +-- 10 files changed, 399 insertions(+), 330 deletions(-) diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 1f55d2fa91..ca9f722890 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -68,9 +68,11 @@ class PresetAgentFailure(CoreModel): class PresetTrialBenchmark(PresetBenchmark): - """A trial's benchmark as the agent recorded it. The preset-quality rules of - `PresetBenchmark.validate_metrics` apply to the final report; a trial may - have failed requests, and the listing still shows what it measured.""" + """A trial's benchmark, which the agent writes into `trials//trial.json` + after benchmarking the trial at a concurrency, read by the CLI for + `dstack preset list`. `PresetBenchmark.validate_metrics` is overridden + because its rule is for the final report; applied to `trial.json` it would + drop the whole trial from the listing.""" @model_validator(mode="after") def validate_metrics(self) -> Self: @@ -79,8 +81,10 @@ def validate_metrics(self) -> Self: class PresetTrialResult(CoreModel): """One `result` entry of `trials//trial.json`, as the system prompt - defines it. `learned` is left out because the agent writes it for its own reading - in later sessions, and not used by the CLI""" + defines it: the agent writes one per concurrency in `constraints.json` once + the trial is benchmarked, and the CLI reads them for `dstack preset list`. + The entry's `learned` field is left out: the CLI does not use it, because + the agent writes it for its own reading in later sessions.""" concurrency: Optional[PositiveInt] = None context_length: Optional[PositiveInt] = None @@ -89,7 +93,9 @@ class PresetTrialResult(CoreModel): class PresetTrial(CoreModel): - """The `trials//trial.json` record as defined in the system_prompt.md.""" + """The `trials//trial.json` record as defined in the system_prompt.md. + The agent writes it once trial `` is benchmarked, and its presence is + what marks the trial completed; the CLI reads it for `dstack preset list`.""" resources: Optional[dict[str, Any]] = None groups: Optional[list[list[dict[str, Any]]]] = None diff --git a/src/dstack/_internal/cli/models/presets.py b/src/dstack/_internal/cli/models/presets.py index 70b9d5c003..851865f3f1 100644 --- a/src/dstack/_internal/cli/models/presets.py +++ b/src/dstack/_internal/cli/models/presets.py @@ -53,8 +53,7 @@ class PresetListOutput(CoreModel): class PresetTrialsSummary(CoreModel): - """What the session row of `dstack preset list` shows about a session's - completed trials.""" + """Built by the CLI for `dstack preset list`.""" count: int # The trial with the highest tok/s that met the constraints. @@ -73,6 +72,8 @@ class PresetTrialsSummary(CoreModel): class PresetSessionEntry(CoreModel): + """Built by the CLI for `dstack preset list`.""" + id: str state: PresetSessionState name: Optional[str] diff --git a/src/tests/_internal/cli/commands/test_preset.py b/src/tests/_internal/cli/commands/test_preset.py index 5c36acd506..8bf3699dd5 100644 --- a/src/tests/_internal/cli/commands/test_preset.py +++ b/src/tests/_internal/cli/commands/test_preset.py @@ -368,8 +368,8 @@ def test_gets_complete_preset_as_json_without_api_client(self, tmp_path, capsys) data = json.loads(capsys.readouterr().out) assert data["id"] == preset.id assert data["created_at"] == render_datetime_as_api(preset.created_at) - assert data["context_length"] == 32768 - assert data["benchmark"]["metrics"]["total_output_tokens"] == 2048 + assert data["result"][0]["context_length"] == 32768 + assert data["result"][0]["benchmark"]["metrics"]["total_output_tokens"] == 2048 @pytest.mark.parametrize( "args", @@ -389,8 +389,8 @@ def test_lists_complete_presets_as_json(self, tmp_path, capsys, args): data = output["presets"][0] assert data["id"] == preset.id assert data["created_at"] == render_datetime_as_api(preset.created_at) - assert data["context_length"] == 32768 - assert data["benchmark"]["metrics"]["total_output_tokens"] == 2048 + assert data["result"][0]["context_length"] == 32768 + assert data["result"][0]["benchmark"]["metrics"]["total_output_tokens"] == 2048 @pytest.mark.parametrize("flag_attribute", [("--base", "base"), ("--repo", "repo")]) def test_deletes_all_presets_of_model_keeping_others_without_api_client( diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index add3afc533..6815c7c963 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -16,11 +16,18 @@ PresetSessionRun, PresetSessionState, PresetSessionWorkspace, + PresetTrialBenchmark, + PresetTrialResult, +) +from dstack._internal.cli.models.presets import ( + PresetSessionEntry, + PresetTrialsSummary, + VerifiedPreset, ) -from dstack._internal.cli.models.presets import VerifiedPreset from dstack._internal.cli.services.presets.agents.base import PresetAgentSpec from dstack._internal.cli.services.presets.store import PresetStore from dstack._internal.compat import IS_WINDOWS +from dstack._internal.core.models.common import validate_extra_ignore from dstack._internal.core.models.configurations import ( DEFAULT_REPLICA_GROUP_NAME, PresetConfiguration, @@ -28,7 +35,9 @@ ) from dstack._internal.core.models.instances import Disk, Gpu, Resources from dstack._internal.core.models.presets import ( + AnyPresetConstraints, PresetBenchmark, + PresetResult, PresetVerificationReplicaGroup, ) from dstack._internal.core.models.resources import ResourcesSpec @@ -169,9 +178,14 @@ def get_preset( "env": ["HF_TOKEN"], } ), - context_length=context_length, best_trial=1, - benchmark=get_preset_benchmark(), + result=[ + PresetResult( + concurrency=1, + context_length=context_length, + benchmark=get_preset_benchmark(), + ) + ], verified_on=[ PresetVerificationReplicaGroup(name=DEFAULT_REPLICA_GROUP_NAME, replicas=[resources]) ], @@ -267,6 +281,82 @@ def get_session_run(**overrides: Any) -> PresetSessionRun: return PresetSessionRun(**fields) +def get_preset_constraints(**overrides: Any) -> AnyPresetConstraints: + """`constraints.json` as a session records it: synthetic by default, a + dataset one when `dataset` is given.""" + fields: dict[str, Any] = { + "run_name_prefix": "qwen-ab12cd34", + "model": {"base": "Qwen/Qwen3.5-27B"}, + "min_context_length": 8192, + "max_ttft": 5000, + "trials_num": 3, + "concurrency": [1], + "baseline": True, + "fleets": ["gpu-fleet"], + "env": [], + } + if "dataset" not in overrides: + fields.update({"input_tokens": 1024, "output_tokens": 1024, "shared_prefix_tokens": 0}) + fields.update(overrides) + return validate_extra_ignore(AnyPresetConstraints, fields) + + +def get_trial_result( + *, + tok_s: float = 42.0, + tpot_ms: Optional[float] = None, + ttft_ms: Optional[float] = None, + concurrency: int = 1, + context_length: Optional[int] = None, + failed: bool = False, +) -> PresetTrialResult: + """One `result` entry of a `trial.json`, with a benchmark that measured + `tok_s` over one second, so it must be a whole number of tokens. `tpot_ms` + sets the mean and `ttft_ms` the p50, the statistics the listing shows.""" + benchmark = get_preset_benchmark().model_dump() + benchmark["workload"]["concurrency"] = concurrency + benchmark["metrics"].update({"duration_seconds": 1.0, "total_output_tokens": tok_s}) + if tpot_ms is not None: + benchmark["metrics"]["tpot_ms"]["mean"] = tpot_ms + if ttft_ms is not None: + benchmark["metrics"]["ttft_ms"]["p50"] = ttft_ms + return PresetTrialResult( + concurrency=concurrency, + context_length=context_length, + benchmark=PresetTrialBenchmark.model_validate(benchmark), + failed=failed, + ) + + +def get_trials_summary(count: int = 0, **overrides: Any) -> PresetTrialsSummary: + """`count` trials that produced no benchmark, unless `series` and `failed` + say otherwise.""" + fields: dict[str, Any] = {"count": count, "series": [None] * count, "failed": [False] * count} + fields.update(overrides) + return PresetTrialsSummary(**fields) + + +def get_session_entry(**overrides: Any) -> PresetSessionEntry: + """A session as `dstack preset list` reads it. `model`, `trials_num` and + `created_at` go to the stored state; the rest are entry fields.""" + state_overrides = { + key: overrides.pop(key) + for key in ("model", "trials_num", "created_at") + if key in overrides + } + fields: dict[str, Any] = { + "id": "ab12cd34", + "name": None, + "status": "running", + "trials": get_trials_summary(), + "verification": None, + "constraints": None, + } + fields.update(overrides) + state = get_session_state(id=fields["id"], status=fields["status"], **state_overrides) + return PresetSessionEntry(state=state, **fields) + + def get_successful_preset_report(run: Run) -> PresetAgentSuccess: return PresetAgentSuccess.model_validate( { @@ -277,8 +367,13 @@ def get_successful_preset_report(run: Run) -> PresetAgentSuccess: "trial": 1, "base": "Qwen/Qwen3.5-27B", "model": "community/Qwen3.5-27B-GPTQ-Int4", - "context_length": 32768, - "benchmark": get_preset_benchmark(), + "result": [ + { + "concurrency": 1, + "context_length": 32768, + "benchmark": get_preset_benchmark(), + } + ], } ) diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index e55c3fd013..87b9434565 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -14,7 +14,12 @@ import pytest import yaml -from dstack._internal.cli.models.preset_agent import PresetAgentInfo, PresetSessionProcess +from dstack._internal.cli.models.preset_agent import ( + PresetAgentInfo, + PresetSessionProcess, + PresetTrial, + PresetTrialResult, +) from dstack._internal.cli.services.presets.agent import ( _prepare_subprocess_command, _terminate_process, @@ -68,6 +73,7 @@ get_fake_api, get_session_run, get_session_state, + get_trial_result, ) @@ -1610,89 +1616,66 @@ def test_refusals(self, tmp_path, monkeypatch, state, match): load_resumable_session(preset_id) -def _write_trials(tmp_path, records): +def _write_trials(tmp_path, trials: list[PresetTrial]): trials_dir = tmp_path / "trials" - for number, record in enumerate(records, 1): + for number, trial in enumerate(trials, 1): (trials_dir / str(number)).mkdir(parents=True) - (trials_dir / str(number) / "trial.json").write_text(json.dumps(record)) + (trials_dir / str(number) / "trial.json").write_text(trial.model_dump_json()) return trials_dir class TestSummarizeSessionTrials: def test_counts_records_even_when_trials_share_a_task(self, tmp_path): - record = { - "resources": {"gpu": {"name": "A40", "memory": "48GB", "count": 1}}, - "benchmark": { - "workload": {"concurrency": 8}, - "metrics": {"duration_seconds": 10.0, "total_output_tokens": 20000}, - }, - } - rerun = json.loads(json.dumps(record)) - rerun["benchmark"]["metrics"]["total_output_tokens"] = 23000 - nameless = {"benchmark": {"workload": {}, "metrics": {}}} - trials_dir = _write_trials(tmp_path, [record, rerun, record, nameless]) + trial = PresetTrial( + resources={"gpu": {"name": "A40", "memory": "48GB", "count": 1}}, + result=[get_trial_result(tok_s=2000.0, concurrency=8)], + ) + rerun = trial.model_copy( + update={"result": [get_trial_result(tok_s=2300.0, concurrency=8)]} + ) + # A trial whose benchmark could not be done, as the agent records it. + unmeasured = PresetTrial(result=[PresetTrialResult(concurrency=8, failed=True)]) + trials_dir = _write_trials(tmp_path, [trial, rerun, trial, unmeasured]) summary = _summarize_session_trials(trials_dir) - assert summary is not None # 4 records = 4 trials: one long-lived task commonly hosts several # trials, so identical records must not collapse the count. - assert summary["count"] == 4 - assert summary["best"] == { - "tok_s": 2300.0, - "tpot_ms": None, - "ttft_ms": None, - "context_length": None, - "concurrency": 8, - "gpu": "A40:48GB:1", - } + assert summary.count == 4 + assert summary.best == rerun + assert summary.gpu == "A40:48GB:1" def test_trials_are_ordered_numerically_beyond_nine(self, tmp_path): # Twelve trials: a string sort would chart 1, 10, 11, 12, 2, ... - records = [ - { - "benchmark": { - "metrics": {"total_output_tokens": float(i * 100), "duration_seconds": 1.0}, - "workload": {"concurrency": 1}, - } - } - for i in range(1, 13) + trials = [ + PresetTrial(result=[get_trial_result(tok_s=float(i * 100))]) for i in range(1, 13) ] - trials_dir = _write_trials(tmp_path, records) + trials_dir = _write_trials(tmp_path, trials) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["series"] == [float(i * 100) for i in range(1, 13)] + assert summary.series == [float(i * 100) for i in range(1, 13)] def test_a_trial_still_in_flight_is_not_counted(self, tmp_path): - record = { - "resources": {}, - "benchmark": { - "workload": {"concurrency": 8}, - "metrics": {"duration_seconds": 10.0, "total_output_tokens": 20000}, - }, - } - trials_dir = _write_trials(tmp_path, [record]) + trial = PresetTrial(resources={}, result=[get_trial_result(tok_s=2000.0, concurrency=8)]) + trials_dir = _write_trials(tmp_path, [trial]) # Trial 2 has started but has no result yet. (trials_dir / "2").mkdir() (trials_dir / "2" / "task.dstack.yml").write_text("type: task\n") summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["count"] == 1 - assert summary["series"] == [2000.0] + assert summary.count == 1 + assert summary.series == [2000.0] def test_a_torn_result_copy_reads_as_still_in_flight(self, tmp_path): trials_dir = _write_trials(tmp_path, []) (trials_dir / "1").mkdir(parents=True) - (trials_dir / "1" / "trial.json").write_text('{"benchmark": {"met') + (trials_dir / "1" / "trial.json").write_text('{"result": [{"benchmark": {"met') summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["count"] == 0 + assert summary.count == 0 class TestReadLastSessionVerification: @@ -1710,7 +1693,7 @@ def test_newest_attempt_wins_and_a_missing_directory_is_not_verifying(self, tmp_ (service_dir / "2" / "service.dstack.yml").write_text("type: service\n") # Attempt 2 has no result yet, so it is the one in flight. - assert _read_last_session_verification(service_dir) == {"status": "verifying"} + assert _read_last_session_verification(service_dir) == "verifying" def test_the_newest_result_is_returned(self, tmp_path): # Attempts 9 and 10: a string sort would pick "9" as the newest. @@ -1721,11 +1704,7 @@ def test_the_newest_result_is_returned(self, tmp_path): json.dumps({"trial": number, "run_name": f"p-{number}", "status": status}) ) - assert _read_last_session_verification(service_dir) == { - "trial": 10, - "run_name": "p-10", - "status": "verified", - } + assert _read_last_session_verification(service_dir) == "verified" def test_a_torn_result_copy_reads_as_verifying(self, tmp_path): # The mirror copies whole files, but a reader can still catch one @@ -1734,7 +1713,7 @@ def test_a_torn_result_copy_reads_as_verifying(self, tmp_path): (service_dir / "1").mkdir(parents=True) (service_dir / "1" / "verification.json").write_text('{"trial": 1, "run_na') - assert _read_last_session_verification(service_dir) == {"status": "verifying"} + assert _read_last_session_verification(service_dir) == "verifying" class TestFileLineReader: diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index c6203023f4..2289f0e137 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -826,7 +826,7 @@ def test_max_ttft_reaches_the_constraints(self): def test_throughput_is_derived_not_read(self): # A miscomputed field must not become the number we rank on. preset = get_preset() - benchmark = preset.benchmark + benchmark = get_or_error(preset.result[0].benchmark) benchmark.metrics.output_tok_per_s = 999999.0 benchmark.metrics.per_user_tok_per_s = 999999.0 @@ -863,7 +863,7 @@ def test_renders_defaults_for_the_optional_fields(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": 8, + "concurrency": [8], "input_tokens": 1024, "output_tokens": 1024, "shared_prefix_tokens": 0, @@ -897,7 +897,7 @@ def test_renders_custom_dataset_without_request_shape(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": 8, + "concurrency": [8], "dataset": "spec_bench", "baseline": True, "fleets": ["gpu-fleet"], @@ -927,7 +927,7 @@ def test_renders_configured_values(self): assert data["model"] == {"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"} assert data["min_context_length"] == 32768 assert data["trials_num"] == 10 - assert data["concurrency"] == 16 + assert data["concurrency"] == [16] assert data["fleets"] == ["a", "b"] @@ -1234,7 +1234,7 @@ def test_lists_a_failed_session(self, tmp_path, monkeypatch): lambda: tmp_path, ) - listed = {entry["id"]: entry["status"] for entry in list_preset_sessions()} + listed = {entry.id: entry.status for entry in list_preset_sessions()} assert listed == {"dead0000": "failed", "beef0000": "success"} diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index e5e963388b..73d703c02a 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -1,15 +1,23 @@ -import json from datetime import datetime, timedelta, timezone from io import StringIO import pytest from rich.table import Table -from dstack._internal.cli.models.presets import PulledPreset +from dstack._internal.cli.models.preset_agent import PresetTrial, PresetTrialResult +from dstack._internal.cli.models.presets import PresetSessionEntry, PulledPreset from dstack._internal.cli.services.presets import output as output_module from dstack._internal.cli.services.presets.output import _add_session, _format_number from dstack._internal.core.models.presets import PortablePreset, PresetWorkload -from tests._internal.cli.common import get_preset, plain_console +from dstack._internal.utils.common import get_or_error +from tests._internal.cli.common import ( + get_preset, + get_preset_constraints, + get_session_entry, + get_trial_result, + get_trials_summary, + plain_console, +) pytestmark = pytest.mark.windows @@ -28,7 +36,7 @@ def test_small_values_keep_three_significant_digits(self): class TestFormatPresetBenchmark: def test_formats_second_scale_ttft_without_scientific_notation(self): preset = get_preset() - ttft = preset.benchmark.metrics.ttft_ms + ttft = get_or_error(preset.result[0].benchmark).metrics.ttft_ms ttft.mean = 8148.3 ttft.p50 = 8151.4 ttft.p99 = 8334.2 @@ -71,7 +79,7 @@ def test_renders_the_requested_workload_not_the_measured_one(self): # They diverge with a dataset, where the benchmark records measured means. preset = get_preset() preset.configuration.dataset = "spec_bench" - benchmark = preset.benchmark + benchmark = get_or_error(preset.result[0].benchmark) benchmark.workload = PresetWorkload( api=benchmark.workload.api, num_requests=benchmark.workload.num_requests, @@ -140,7 +148,7 @@ def test_prints_submitted_column(self, monkeypatch): assert "2 months ago" in output.getvalue() -def _session_row(session: dict) -> dict: +def _session_row(session: PresetSessionEntry) -> dict: table = Table(box=None) for column in ( "BASE", @@ -163,20 +171,17 @@ def _session_row(session: dict) -> dict: class TestSessionRow: def test_shows_progress_after_status_and_best_benchmark(self): row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "trials_num": 3, - "trials": { - "count": 2, - "best": { - "tok_s": 2339.0, - "tpot_ms": 3.42, - "concurrency": 8, - "gpu": "A40:48GB:1", - }, - }, - } + get_session_entry( + id="c7e18d52", + trials_num=3, + trials=get_trials_summary( + 2, + best=PresetTrial( + result=[get_trial_result(tok_s=2339.0, tpot_ms=3.42, concurrency=8)] + ), + gpu="A40:48GB:1", + ), + ) ) # 2 completed, so the 3rd is the one being trialed. @@ -188,27 +193,25 @@ def test_shows_progress_after_status_and_best_benchmark(self): def test_shows_the_shared_prefix_when_the_workload_has_one(self): row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "constraints": { - "input_tokens": 8192, - "output_tokens": 1024, - "concurrency": 162, - "shared_prefix_tokens": 7373, - }, - } + get_session_entry( + id="c7e18d52", + constraints=get_preset_constraints( + input_tokens=8192, + output_tokens=1024, + concurrency=[162], + shared_prefix_tokens=7373, + ), + ) ) assert row["CONSTRAINTS"] == ("[secondary]io=8K/1K prefix=90% c=162[/]") def test_shows_the_dataset_for_a_session_with_a_custom_dataset(self): row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "constraints": {"dataset": "spec_bench", "concurrency": 4}, - } + get_session_entry( + id="c7e18d52", + constraints=get_preset_constraints(dataset="spec_bench", concurrency=[4]), + ) ) assert row["CONSTRAINTS"] == ("[secondary]data=spec_bench c=4[/]") @@ -217,47 +220,38 @@ def test_omits_the_shared_prefix_when_requests_are_fully_unique(self): # A zero prefix is the default; only a non-zero share is information. # can serve from cache, so a row without it cannot be compared to one with. row = _session_row( - { - "id": "c7e18d52", - "status": "running", - "constraints": { - "input_tokens": 8192, - "output_tokens": 1024, - "concurrency": 162, - "shared_prefix_tokens": 0, - }, - } + get_session_entry( + id="c7e18d52", + constraints=get_preset_constraints( + input_tokens=8192, + output_tokens=1024, + concurrency=[162], + shared_prefix_tokens=0, + ), + ) ) assert "prefix" not in row["CONSTRAINTS"] def test_shows_zero_progress_without_benchmark(self): - row = _session_row( - {"id": "ab12cd34", "status": "running", "trials_num": 3, "trials": {"count": 0}} - ) + row = _session_row(get_session_entry(trials_num=3)) assert row["STATUS"] == "[bold sea_green3]trialing[/] [secondary](1/3)[/]" def test_verifying_keeps_the_completed_count(self): row = _session_row( - { - "id": "ab12cd34", - "status": "running", - "trials_num": 3, - "trials": {"count": 3}, - "verification": {"status": "verifying"}, - } + get_session_entry(trials_num=3, trials=get_trials_summary(3), verification="verifying") ) assert row["STATUS"] == "[bold deep_sky_blue1]verifying[/] [secondary](3/3)[/]" def test_omits_progress_without_trials_data(self): - row = _session_row({"id": "ab12cd34", "status": "interrupted"}) + row = _session_row(get_session_entry(status="interrupted")) assert row["STATUS"] == "[secondary]interrupted[/]" def test_counts_without_trials_num(self): - row = _session_row({"id": "ab12cd34", "status": "interrupted", "trials": {"count": 2}}) + row = _session_row(get_session_entry(status="interrupted", trials=get_trials_summary(2))) assert row["STATUS"] == "[secondary]interrupted[/] [secondary](2)[/]" @@ -271,18 +265,18 @@ def test_sorts_all_rows_newest_first_without_grouping(self, monkeypatch): update={"id": "11aa22bb", "created_at": old.created_at + timedelta(days=2)} ) sessions = [ - { - "id": "aaaaaaaa", - "status": "interrupted", - "model": old.base, - "created_at": "2026-07-01T00:00:00Z", - }, - { - "id": "bbbbbbbb", - "status": "interrupted", - "model": old.base, - "created_at": "2026-07-02T00:00:00Z", - }, + get_session_entry( + id="aaaaaaaa", + status="interrupted", + model=old.base, + created_at=datetime(2026, 7, 1, tzinfo=timezone.utc), + ), + get_session_entry( + id="bbbbbbbb", + status="interrupted", + model=old.base, + created_at=datetime(2026, 7, 2, tzinfo=timezone.utc), + ), ] output_module.print_presets([old, new], sessions=sessions) @@ -311,13 +305,13 @@ def test_saved_preset_owns_row_regardless_of_session_status(self, monkeypatch, s monkeypatch.setattr(output_module, "console", plain_console(buffer, width=200)) preset = get_preset() sessions = [ - { - "id": preset.id, - "status": session_status, - "model": preset.base, - "trials_num": 4, - "trials": {"count": 3}, - } + get_session_entry( + id=preset.id, + status=session_status, + model=preset.base, + trials_num=4, + trials=get_trials_summary(3), + ) ] output_module.print_presets([preset], sessions=sessions) @@ -331,13 +325,9 @@ def test_saved_preset_owns_row_regardless_of_session_status(self, monkeypatch, s class TestVerifyingStatus: def test_running_session_with_a_verification_record_shows_verifying(self): row = _session_row( - { - "id": "ab12cd34", - "status": "running", - "trials_num": 10, - "trials": {"count": 6}, - "verification": {"trial": 3, "run_name": "p-2", "status": "verifying"}, - } + get_session_entry( + trials_num=10, trials=get_trials_summary(6), verification="verifying" + ) ) # The trial budget is not spent, which the old inference required. @@ -345,30 +335,22 @@ def test_running_session_with_a_verification_record_shows_verifying(self): def test_a_failed_attempt_still_counts_as_verifying(self): row = _session_row( - { - "id": "ab12cd34", - "status": "running", - "trials_num": 2, - "trials": {"count": 2}, - "verification": {"trial": 3, "status": "failed", "reason": "probe never passed"}, - } + get_session_entry(trials_num=2, trials=get_trials_summary(2), verification="failed") ) assert row["STATUS"].startswith("[bold deep_sky_blue1]verifying[/]") def test_a_spent_trial_budget_alone_stays_trialing(self): - row = _session_row( - {"id": "ab12cd34", "status": "running", "trials_num": 2, "trials": {"count": 2}} - ) + row = _session_row(get_session_entry(trials_num=2, trials=get_trials_summary(2))) assert row["STATUS"].startswith("[bold sea_green3]trialing[/]") -def _write_trials(tmp_path, records): +def _write_trials(tmp_path, trials: list[PresetTrial]): trials_dir = tmp_path / "trials" - for number, record in enumerate(records, 1): + for number, trial in enumerate(trials, 1): (trials_dir / str(number)).mkdir(parents=True) - (trials_dir / str(number) / "trial.json").write_text(json.dumps(record)) + (trials_dir / str(number) / "trial.json").write_text(trial.model_dump_json()) return trials_dir @@ -378,28 +360,19 @@ def test_a_failed_trial_keeps_its_benchmark_but_never_becomes_best(self, tmp_pat # The failed trial is the fastest. It broke `max_ttft`, so promoting it # would put a non-compliant configuration at the top of the listing. - trials_dir = _write_trials( - tmp_path, - [ - { - "benchmark": { - "metrics": {"total_output_tokens": tokens, "duration_seconds": 1.0}, - "workload": {"concurrency": 8}, - }, - **({"failed": True} if failed else {}), - } - for tokens, failed in ((100.0, False), (900.0, True), (300.0, False)) - ], - ) + trials = [ + PresetTrial(result=[get_trial_result(tok_s=tokens, concurrency=8, failed=failed)]) + for tokens, failed in ((100.0, False), (900.0, True), (300.0, False)) + ] + trials_dir = _write_trials(tmp_path, trials) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["count"] == 3 + assert summary.count == 3 # Still charted: the trial happened and its number is real. - assert summary["series"] == [100.0, 900.0, 300.0] - assert summary["best"]["tok_s"] == 300.0 - assert summary["best_failed"]["tok_s"] == 900.0 + assert summary.series == [100.0, 900.0, 300.0] + assert summary.best == trials[2] + assert summary.best_failed == trials[1] def test_reports_the_gpu_when_no_trial_produced_a_benchmark(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials @@ -408,28 +381,36 @@ def test_reports_the_gpu_when_no_trial_produced_a_benchmark(self, tmp_path): # the only thing left that knows what the run was on. trials_dir = _write_trials( tmp_path, - [{"resources": {"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}}], + [ + PresetTrial( + resources={"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}, + result=[PresetTrialResult(concurrency=4, failed=True)], + ) + ], ) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["best"] is None - assert summary["best_failed"] is None - assert summary["gpu"] == "MI300X:192GB:1" + assert summary.best is None + assert summary.best_failed is None + assert summary.gpu == "MI300X:192GB:1" def test_reports_the_gpu_of_a_single_node_group(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials trials_dir = _write_trials( tmp_path, - [{"groups": [[{"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}]]}], + [ + PresetTrial( + groups=[[{"gpu": {"name": "MI300X", "memory": "192GB", "count": 1}}]], + result=[get_trial_result()], + ) + ], ) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["gpu"] == "MI300X:192GB:1" + assert summary.gpu == "MI300X:192GB:1" def test_counts_the_worker_nodes_of_a_disaggregated_trial(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials @@ -438,13 +419,17 @@ def test_counts_the_worker_nodes_of_a_disaggregated_trial(self, tmp_path): h200 = {"gpu": {"name": "H200", "memory": "141GB", "count": 8}} trials_dir = _write_trials( tmp_path, - [{"groups": [[{"cpu": "16"}], [h200], [h200, h200]]}], + [ + PresetTrial( + groups=[[{"cpu": "16"}], [h200], [h200, h200]], + result=[get_trial_result()], + ) + ], ) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["gpu"] == "H200:141GB:8 x3" + assert summary.gpu == "H200:141GB:8 x3" def test_shows_the_split_when_roles_ran_different_gpus(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials @@ -453,72 +438,64 @@ def test_shows_the_split_when_roles_ran_different_gpus(self, tmp_path): trials_dir = _write_trials( tmp_path, [ - { - "groups": [ + PresetTrial( + groups=[ [{"gpu": {"name": "H200", "memory": "141GB", "count": 8}}], [h100, h100], - ] - } + ], + result=[get_trial_result()], + ) ], ) summary = _summarize_session_trials(trials_dir) - assert summary is not None # Group order, not sorted: the roles read in the order they ran. - assert summary["gpu"] == "H200:141GB:8 + H100:80GB:8 x2" + assert summary.gpu == "H200:141GB:8 + H100:80GB:8 x2" def test_the_fastest_failed_trial_is_kept_when_nothing_passed(self, tmp_path): from dstack._internal.cli.services.presets.session import _summarize_session_trials - trials_dir = _write_trials( - tmp_path, - [ - { - "benchmark": { - "metrics": { - "total_output_tokens": tokens, - "duration_seconds": 1.0, - "tpot_ms": {"mean": 41.7, "p50": 34.4}, - "ttft_ms": {"p50": 4300.0}, - }, - "workload": {"concurrency": 4}, - }, - "failed": True, - } - for tokens in (100.0, 300.0, 200.0) - ], - ) + trials = [ + PresetTrial( + result=[ + get_trial_result( + tok_s=tokens, tpot_ms=41.7, ttft_ms=4300.0, concurrency=4, failed=True + ) + ] + ) + for tokens in (100.0, 300.0, 200.0) + ] + trials_dir = _write_trials(tmp_path, trials) summary = _summarize_session_trials(trials_dir) - assert summary is not None - assert summary["best"] is None - assert summary["best_failed"]["tok_s"] == 300.0 - # Mean, not p50: MTP skews TPOT right, and mean is the delivered rate. - assert summary["best_failed"]["tpot_ms"] == 41.7 + assert summary.best is None + assert summary.best_failed == trials[1] def test_a_run_that_met_nothing_still_shows_what_it_measured(self): row = _session_row( - { - "id": "ab12cd34", - "status": "running", - "constraints": {"input_tokens": 10000, "output_tokens": 1500}, - "trials": { - "count": 1, - "series": [100.0], - "failed": [True], - "best": None, - "best_failed": { - "tok_s": 100.0, - "tpot_ms": 34.4, - "ttft_ms": 4300.0, - "context_length": 1048576, - "concurrency": 4, - "gpu": "MI300X:192GB:1", - }, - }, - } + get_session_entry( + constraints=get_preset_constraints(input_tokens=10000, output_tokens=1500), + trials=get_trials_summary( + 1, + series=[100.0], + failed=[True], + best_failed=PresetTrial( + result=[ + get_trial_result( + tok_s=100.0, + tpot_ms=34.4, + ttft_ms=4300.0, + concurrency=4, + context_length=1048576, + failed=True, + ) + ] + ), + gpu="MI300X:192GB:1", + ), + ) ) assert "ttft=4.3s" in row["BENCHMARK"] @@ -532,15 +509,9 @@ class TestFailedTrialSpark: def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): # Its number is real, so it earns a bar rather than a `·`, and green goes # to the best trial that meets the constraints even on a lower number. - session = { - "id": "ab12cd34", - "status": "running", - "trials": { - "count": 3, - "series": [100.0, 900.0, 300.0], - "failed": [False, True, False], - }, - } + session = get_session_entry( + trials=get_trials_summary(3, series=[100.0, 900.0, 300.0], failed=[False, True, False]) + ) spark = output_module._format_trial_spark(session) @@ -552,15 +523,9 @@ def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): def test_the_best_failed_trial_is_gold_while_none_passes(self): # With nothing meeting the constraints, the best result so far is still # what the run has to show; the rest are context. - session = { - "id": "ab12cd34", - "status": "running", - "trials": { - "count": 3, - "series": [100.0, 900.0, 300.0], - "failed": [True, True, True], - }, - } + session = get_session_entry( + trials=get_trials_summary(3, series=[100.0, 900.0, 300.0], failed=[True, True, True]) + ) spark = output_module._format_trial_spark(session) diff --git a/src/tests/_internal/cli/services/presets/test_report_schema.py b/src/tests/_internal/cli/services/presets/test_report_schema.py index 9298e06d2a..4fc79f8734 100644 --- a/src/tests/_internal/cli/services/presets/test_report_schema.py +++ b/src/tests/_internal/cli/services/presets/test_report_schema.py @@ -7,6 +7,7 @@ ) from dstack._internal.cli.services.presets.verify import _AGENT_RESULT_ADAPTER from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.utils.common import get_or_error pytestmark = pytest.mark.windows @@ -18,8 +19,7 @@ "trial": None, "base": None, "model": None, - "context_length": None, - "benchmark": None, + "result": None, "failure_summary": None, } # Keywords the OpenAI strict structured-output mode rejects. @@ -105,32 +105,37 @@ def test_keeps_the_agent_facing_shape(self): "trial": 1, "base": "Qwen/Qwen2.5-0.5B-Instruct", "model": "Qwen/Qwen2.5-0.5B-Instruct", - "context_length": 2048, - "benchmark": { - "tool": "vllm.bench serve", - "tool_version": "0.27.0", - "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", - "workload": { - "api": "chat_completions", - "dataset": None, - "num_requests": 32, - "input_tokens": 128, - "output_tokens": 64, + "result": [ + { "concurrency": 1, - "shared_prefix_tokens": 0, - }, - "metrics": { - "successful_requests": 32, - "failed_requests": 0, - "duration_seconds": 40.5, - "total_input_tokens": 4096, - "total_output_tokens": 2048, - "output_tok_per_s": 50.6, - "per_user_tok_per_s": 50.6, - "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, - "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, - }, - }, + "context_length": 2048, + "benchmark": { + "tool": "vllm.bench serve", + "tool_version": "0.27.0", + "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", + "workload": { + "api": "chat_completions", + "dataset": None, + "num_requests": 32, + "input_tokens": 128, + "output_tokens": 64, + "concurrency": 1, + "shared_prefix_tokens": 0, + }, + "metrics": { + "successful_requests": 32, + "failed_requests": 0, + "duration_seconds": 40.5, + "total_input_tokens": 4096, + "total_output_tokens": 2048, + "output_tok_per_s": 50.6, + "per_user_tok_per_s": 50.6, + "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, + "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, + }, + }, + } + ], }, PresetAgentSuccess, ), @@ -146,5 +151,6 @@ def test_reports_codex_produced_against_the_schema_parse(self, data, expected): assert isinstance(report, expected) if isinstance(report, PresetAgentSuccess): assert isinstance(report.service_yaml, ServiceConfiguration) - assert report.benchmark.workload.dataset is None - assert report.benchmark.workload.shared_prefix_tokens == 0 + workload = get_or_error(report.result[0].benchmark).workload + assert workload.dataset is None + assert workload.shared_prefix_tokens == 0 diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index fb774a68ff..f3a1ee92cd 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -114,14 +114,14 @@ def test_loads_a_preset_that_requested_the_random_dataset(self, tmp_path: Path): path = tmp_path / "presets" / preset.id / "preset.yml" data = yaml.safe_load(path.read_text()) data["configuration"]["dataset"] = "random" - data["benchmark"]["workload"]["dataset"] = "random" - data["benchmark"]["workload"]["shared_prefix_tokens"] = 0 + data["result"][0]["benchmark"]["workload"]["dataset"] = "random" + data["result"][0]["benchmark"]["workload"]["shared_prefix_tokens"] = 0 path.write_text(yaml.safe_dump(data, sort_keys=False)) loaded = get_verified_preset(store, preset.id) assert loaded.configuration.dataset is None - assert loaded.benchmark.workload.dataset is None + assert get_or_error(loaded.result[0].benchmark).workload.dataset is None def test_loads_a_preset_that_predates_the_repo_field(self, tmp_path: Path): # Presets written before the rename store the served repo as `model`. @@ -226,7 +226,7 @@ def test_reports_one_arms_errors_for_an_unreadable_preset(self, tmp_path: Path): store.save(preset) path = tmp_path / "presets" / preset.id / "preset.yml" data = yaml.safe_load(path.read_text()) - del data["benchmark"] + del data["result"] path.write_text(yaml.safe_dump(data, sort_keys=False)) with pytest.raises(CLIError) as excinfo: @@ -236,7 +236,7 @@ def test_reports_one_arms_errors_for_an_unreadable_preset(self, tmp_path: Path): # Every reported error is located under the tag the file claims; the # untagged union used to add the other arm's errors on top. assert "1 validation error for" in message - assert "verified.benchmark" in message + assert "verified.result" in message assert "published." not in message def test_saving_same_id_overwrites_existing_preset(self, tmp_path: Path): @@ -244,7 +244,12 @@ def test_saving_same_id_overwrites_existing_preset(self, tmp_path: Path): preset = get_preset() store.save(preset) - updated = preset.model_copy(update={"base": "Qwen/Another-Model", "context_length": 16384}) + updated = preset.model_copy( + update={ + "base": "Qwen/Another-Model", + "result": [preset.result[0].model_copy(update={"context_length": 16384})], + } + ) store.save(updated) assert store.get(preset.id) == updated @@ -289,7 +294,7 @@ def test_skips_invalid_preset_on_list_but_keeps_it_deletable(self, tmp_path: Pat store.save(valid) path = store.save(get_preset()) data = yaml.safe_load(path.read_text()) - data["benchmark"].pop("metrics") + data["result"][0]["benchmark"].pop("metrics") path.write_text(yaml.safe_dump(data, sort_keys=False)) # One corrupt file must not take down every read; the warning goes to @@ -383,7 +388,7 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert preset.configuration.input_tokens == 10000 assert preset.configuration.output_tokens == 1500 assert preset.configuration.shared_prefix_tokens == 0 - assert preset.configuration.concurrency == 4 + assert preset.configuration.concurrency == [4] # `priority` is not an excluded field, so the regrouping keeps it. assert preset.service.priority == 0 assert [group.name for group in preset.verified_on] == [ @@ -391,7 +396,7 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): ] assert get_or_error(preset.verified_on[0].replicas[0].gpu).name == ["MI300X"] # The old format never recorded a dataset name for a synthetic workload. - assert preset.benchmark.workload.dataset is None + assert get_or_error(preset.result[0].benchmark).workload.dataset is None assert store.list() == [preset] def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index c7a4d9ea80..99b2e51123 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -21,7 +21,7 @@ from dstack._internal.core.models.configurations import PresetConfiguration from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.files import FilePathMapping -from dstack._internal.core.models.presets import PresetWorkload +from dstack._internal.core.models.presets import PresetResult, PresetWorkload from dstack._internal.core.models.profiles import ProfileParams from dstack._internal.core.models.runs import Run from dstack._internal.utils.common import get_or_error @@ -60,8 +60,9 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): verification_replica_groups=base.verified_on, base_model="Qwen/Qwen3.5-27B", repo="community/Qwen3.5-27B-GPTQ-Int4", - context_length=32768, - benchmark=get_preset_benchmark(), + result=[ + PresetResult(concurrency=1, context_length=32768, benchmark=get_preset_benchmark()) + ], configuration=configuration, best_trial=1, preset_id="8f3a12c4", @@ -76,12 +77,12 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): assert preset.configuration.spot_policy is None assert "secret" not in preset.model_dump_json() - def test_successful_report_requires_benchmark(self): + def test_successful_report_requires_result(self): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() - data.pop("benchmark") + data.pop("result") - with pytest.raises(ValidationError, match="benchmark"): + with pytest.raises(ValidationError, match="result"): validate_extra_ignore(AnyPresetAgentResult, data) def test_builds_portable_self_contained_preset(self, tmp_path): @@ -109,7 +110,7 @@ def test_builds_portable_self_contained_preset(self, tmp_path): assert preset.base == "Qwen/Qwen3.5-27B" assert preset.repo == "community/Qwen3.5-27B-GPTQ-Int4" - assert preset.context_length == 32768 + assert preset.result[0].context_length == 32768 assert preset.created_at == created_at assert preset.service.name is None assert preset.service.gateway is None @@ -202,8 +203,9 @@ def test_verifies_a_shared_prefix_benchmark_named_by_the_benchmark_tool(self, tm # The stored workload states the request: no dataset was asked for, so # none is recorded — the tool's own name for its generated data stays in # `command` — while the requested prefix survives into the record. - assert preset.benchmark.workload.dataset is None - assert preset.benchmark.workload.shared_prefix_tokens == 130048 + workload = get_or_error(preset.result[0].benchmark).workload + assert workload.dataset is None + assert workload.shared_prefix_tokens == 130048 def test_rejects_a_benchmark_without_the_requested_shared_prefix(self, tmp_path): run = get_running_service_run() @@ -226,7 +228,7 @@ def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): run = get_running_service_run() with pytest.raises( - CLIError, match="concurrency of 8 does not match the requested concurrency of 4" + CLIError, match=r"concurrency of 8 does not match the requested concurrency of \[4\]" ): build_verified_preset( run=run, @@ -256,7 +258,7 @@ def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - assert preset.benchmark.workload.dataset == "spec_bench" + assert get_or_error(preset.result[0].benchmark).workload.dataset == "spec_bench" def test_stores_a_dataset_benchmark_without_a_stray_shared_prefix(self, tmp_path): # The mirror of the synthetic case: a dataset defines its own requests, @@ -282,8 +284,9 @@ def test_stores_a_dataset_benchmark_without_a_stray_shared_prefix(self, tmp_path created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - assert preset.benchmark.workload.dataset == "spec_bench" - assert preset.benchmark.workload.shared_prefix_tokens == 0 + workload = get_or_error(preset.result[0].benchmark).workload + assert workload.dataset == "spec_bench" + assert workload.shared_prefix_tokens == 0 # A named dataset is the one dataset name the report and the request share, so # the report either echoes it or does not answer the request. @@ -346,22 +349,23 @@ def test_redacts_known_secret_in_benchmark_command_instead_of_failing(self, tmp_ run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["result"][0]["benchmark"]["command"] = ( "python bench.py --header 'Authorization: Bearer sk-live-0123456789abcdef'" ) report = self._load(tmp_path, data, redacted_values=("sk-live-0123456789abcdef",)) assert isinstance(report, PresetAgentSuccess) - assert report.benchmark is not None - assert report.benchmark.command.endswith("Bearer [redacted]'") - assert "sk-live" not in report.benchmark.command + benchmark = report.result[0].benchmark + assert benchmark is not None + assert benchmark.command.endswith("Bearer [redacted]'") + assert "sk-live" not in benchmark.command def test_still_rejects_unknown_bearer_token(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["result"][0]["benchmark"]["command"] = ( "curl -H 'Authorization: Bearer sk-unknown-9876543210fedcba'" ) @@ -374,7 +378,7 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["result"][0]["benchmark"]["command"] = ( "./benchenv/bin/python bench_service.py --base $DSTACK_SERVER_URL/x" " (auth via DSTACK_TOKEN bearer header from env)" ) @@ -382,8 +386,9 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): report = self._load(tmp_path, data, redacted_values=()) assert isinstance(report, PresetAgentSuccess) - assert report.benchmark is not None - assert "bearer header" in report.benchmark.command + benchmark = report.result[0].benchmark + assert benchmark is not None + assert "bearer header" in benchmark.command def _shared_prefix_configuration() -> PresetConfiguration: @@ -425,4 +430,11 @@ def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: def _report_with_workload(run: Run, workload: dict[str, Any]) -> PresetAgentSuccess: benchmark = get_preset_benchmark() benchmark.workload = PresetWorkload.model_validate(workload) - return get_successful_preset_report(run).model_copy(update={"benchmark": benchmark}) + # The entry states the same concurrency as its benchmark, so the test + # exercises the request check and not the entry/benchmark consistency one. + entry = PresetResult( + concurrency=benchmark.workload.concurrency, + context_length=32768, + benchmark=benchmark, + ) + return get_successful_preset_report(run).model_copy(update={"result": [entry]}) From 09ffc450aff08b183674547ecd74dbb6c3b4eaed Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Wed, 18 Nov 2026 13:56:06 +0545 Subject: [PATCH 09/12] Add new tests for concurrency sweep --- .../cli/services/presets/test_output.py | 77 +++++++++++++++++++ .../cli/services/presets/test_verify.py | 60 +++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index 73d703c02a..93428705e3 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -505,6 +505,83 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): assert "*tps/user" in row["BENCHMARK"] +class TestSweepRow: + def test_shows_the_passing_sweep_of_the_best_trial_as_ranges(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + # The trial broke `max_ttft` at 192 only, so the row ranges over 48 and 96. + trial = PresetTrial( + result=[ + get_trial_result( + tok_s=100, tpot_ms=10.0, ttft_ms=400.0, concurrency=48, context_length=32768 + ), + get_trial_result( + tok_s=180, tpot_ms=20.0, ttft_ms=800.0, concurrency=96, context_length=32768 + ), + get_trial_result( + tok_s=200, + tpot_ms=40.0, + ttft_ms=6000.0, + concurrency=192, + context_length=32768, + failed=True, + ), + ] + ) + summary = _summarize_session_trials(_write_trials(tmp_path, [trial])) + + row = _session_row( + get_session_entry( + trials=summary, constraints=get_preset_constraints(concurrency=[48, 96, 192]) + ) + ) + + assert row["CONSTRAINTS"] == "[secondary]io=1K/1K c=48..96[/]" + assert row["BENCHMARK"] == "tps/user=50..100 ttft=400ms..800ms ctx=32K" + + def test_falls_back_to_the_whole_failed_sweep_when_nothing_passed(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + trial = PresetTrial( + result=[ + get_trial_result( + tok_s=100, + tpot_ms=10.0, + ttft_ms=400.0, + concurrency=48, + context_length=32768, + failed=True, + ), + get_trial_result( + tok_s=180, + tpot_ms=20.0, + ttft_ms=800.0, + concurrency=96, + context_length=32768, + failed=True, + ), + get_trial_result( + tok_s=200, + tpot_ms=40.0, + ttft_ms=6000.0, + concurrency=192, + context_length=32768, + failed=True, + ), + ] + ) + summary = _summarize_session_trials(_write_trials(tmp_path, [trial])) + + row = _session_row( + get_session_entry( + trials=summary, constraints=get_preset_constraints(concurrency=[48, 96, 192]) + ) + ) + + assert row["CONSTRAINTS"] == "[secondary]io=1K/1K c=48..192[/]" + assert row["BENCHMARK"] == "[secondary]*tps/user=25..100 ttft=400ms..6s ctx=32K[/]" + + class TestFailedTrialSpark: def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): # Its number is real, so it earns a bar rather than a `·`, and green goes diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index 99b2e51123..e3944f1efc 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -241,6 +241,39 @@ def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) + def test_accepts_a_sweep_benchmarked_at_some_concurrencies_only(self, tmp_path): + # Of the requested 48, 96 and 192, the agent could benchmark 48 only. + run = get_running_service_run() + + preset = build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=_sweep_report(run, measured=(48,)), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert [r.concurrency for r in preset.result] == [48, 96, 192] + assert [r.benchmark is not None for r in preset.result] == [True, False, False] + + def test_rejects_a_sweep_with_no_completed_benchmark(self, tmp_path): + run = get_running_service_run() + + with pytest.raises(CLIError, match="no completed benchmark"): + build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=_sweep_report(run, measured=()), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): run = get_running_service_run() report = _dataset_report(run, dataset="spec_bench") @@ -427,6 +460,33 @@ def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: ) +def _sweep_configuration() -> PresetConfiguration: + return PresetConfiguration.model_validate( + { + "type": "preset", + "base": "Qwen/Qwen3.5-27B", + "min_context_length": 8192, + "max_ttft": 5000, + "trials": 2, + "concurrency": [48, 96, 192], + } + ) + + +def _sweep_report(run: Run, *, measured: tuple[int, ...]) -> PresetAgentSuccess: + """A report over the sweep 48, 96, 192 with a benchmark at `measured` only.""" + result = [] + for concurrency in (48, 96, 192): + benchmark = None + if concurrency in measured: + benchmark = get_preset_benchmark() + benchmark.workload = benchmark.workload.model_copy(update={"concurrency": concurrency}) + result.append( + PresetResult(concurrency=concurrency, context_length=32768, benchmark=benchmark) + ) + return get_successful_preset_report(run).model_copy(update={"result": result}) + + def _report_with_workload(run: Run, workload: dict[str, Any]) -> PresetAgentSuccess: benchmark = get_preset_benchmark() benchmark.workload = PresetWorkload.model_validate(workload) From 9a06a7048315348939deef9785f8c14711749194 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Wed, 18 Nov 2026 20:47:48 +0545 Subject: [PATCH 10/12] Update Preset Docs and fix verification gaps --- mkdocs/docs/concepts/presets.md | 31 +++++----- .../_internal/cli/models/preset_agent.py | 27 ++++++++- .../_internal/cli/services/presets/output.py | 46 +++++++-------- .../_internal/cli/services/presets/session.py | 18 +++--- .../_internal/cli/services/presets/verify.py | 28 ++++++---- .../cli/services/presets/test_agent.py | 41 ++++++++++++++ .../cli/services/presets/test_verify.py | 56 +++++++++++++++++++ 7 files changed, 190 insertions(+), 57 deletions(-) diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index b01c63f62b..7f78972314 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -33,8 +33,8 @@ trials: 5 min_context_length: 1048576 max_ttft: 675 -# The number of simultaneous requests every benchmark uses -concurrency: 1 +# A list of simultaneous request counts every benchmark uses. +concurrency: [4, 8, 16] # The request shape every benchmark uses (defaults to 1024 and 1024) input_tokens: 10000 @@ -55,7 +55,7 @@ To create the preset, pass the configuration to the `dstack apply` command: $ dstack apply -f preset.dstack.yml --fleet b200-fleet Create the preset dsv4-flash? [y/n]: y [2026-08-04 11:38:34] Starting preset creation for deepseek-ai/DeepSeek-V4-Flash. Allowed fleets: b200-fleet. -[2026-08-04 12:31:19] Trial 3 switched from vLLM to SGLang: 319 tok/s per user, 2.2x the baseline. +[2026-08-04 12:31:19] Trial 3 switched from vLLM to SGLang: 319/495/570 tok/s per user at concurrency 4/8/16, 2.1x/1.9x/1.7x the baseline. [2026-08-04 13:04:52] Final service dsv4-flash-c83375b4-4 verified with context length 1048576. [2026-08-04 13:12:07] Benchmark via sglang.bench_serving: 32/32 requests succeeded. ``` @@ -255,9 +255,9 @@ A pulled preset works like any other, and is named `/`: ```shell $ dstack preset list -a - NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED - main/dsv4-flash-b200 8f065dde deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tps/user=309 ttft=213ms ctx=1M pulled 2 min ago - qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tps/user=19.6 ttft=3.43s ctx=32K verified (7) 3 days ago + NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED + main/dsv4-flash-b200 8f065dde deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=4..16 tps/user=319..570 ttft=213ms..465ms ctx=1M pulled 2 min ago + qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tps/user=19.6 ttft=3.43s ctx=32K verified (7) 3 days ago ``` @@ -336,8 +336,8 @@ Use `dstack preset` to list presets: ```shell $ dstack preset list - NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED - dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tps/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED + dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=4..16 tps/user=319..570 ttft=213ms..465ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago ``` @@ -348,18 +348,20 @@ By default, `dstack preset` shows creations that are still running, or the most ```shell $ dstack preset list -a - NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED - dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=1 tps/user=309 ttft=213ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago - qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=64 tps/user=19.6 ttft=3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago - qwen36-pro4500 9ab0fa65 Qwen/Qwen3.6-27B io=1K/1K c=8 tps/user=57.1 ttft=499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago - qwen3-32b-5090 f91d6b60 Qwen/Qwen3-32B io=1K/512 c=8 tps/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago + NAME ID BASE CONSTRAINTS BENCHMARK STATUS SUBMITTED + dsv4-flash-b200 c83375b4 deepseek-ai/DeepSeek-V4-Flash io=10K/1.5K c=4..16 tps/user=319..570 ttft=213ms..465ms ctx=1M ▂▁██▇ trialing (5/5) 2 min ago + qwen35-pro6000 092c792b Qwen/Qwen3.5-397B-A17B io=8K/1K c=16..64 tps/user=19.6..41.2 ttft=1.21s..3.43s ctx=32K ▁▂▅▇█·· verified (7) 3 days ago + qwen36-pro4500 9ab0fa65 Qwen/Qwen3.6-27B io=1K/1K c=2..8 tps/user=57.1..94.3 ttft=312ms..499ms ctx=128K ▁▄██▆·█ verified (7) 4 days ago + qwen3-32b-5090 f91d6b60 Qwen/Qwen3-32B io=1K/512 c=8 tps/user=85.8 ttft=368ms ctx=32K ▁▁▅▅▄▅▇▄▇█ verified (10) 2 weeks ago ``` The `CONSTRAINTS` column is what the creation was asked for, and `BENCHMARK` is the best trial so far. `tps/user` is the steady decode rate, measured as one second divided by the mean time per output token, so it excludes the time to the first token. -The glyphs after the benchmark are one per trial: height is throughput, a yellow bar is a trial whose benchmark broke a constraint, and a red `·` is one that produced no benchmark at all. The shape shows whether a run converged or wandered. +When `concurrency` is a list, `CONSTRAINTS` shows it as `c=min..max`, and the `BENCHMARK` metrics become ranges over the concurrencies that met the constraints, e.g. `tps/user=min..max` and `ttft=min..max`. + +The glyphs after the benchmark are one per trial: height is the trial's highest throughput across its concurrency values that met the constraints, a yellow bar is a trial that broke a constraint at every concurrency, and a red `·` is one that produced no benchmark at all. The shape shows whether a run converged or wandered. Pass `-w` to watch in realtime, `-v` for more detail, or `--json` for complete preset objects. Filter with `--base` or `--repo`. @@ -390,7 +392,6 @@ At the same time, it's recommended to create presets using your own agent — ei ## Limitations * Currently, the agent doesn't upload compiled binaries anywhere; patches compile at runtime -* Doesn't support ranges for `concurrency` > Report bugs and request features on [GitHub](https://github.com/dstackai/dstack/issues), and ask questions on [Discord](https://discord.gg/u8SmfwPpMd). diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index dc2e6daadd..33d6128121 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -90,6 +90,23 @@ class PresetTrialResult(CoreModel): benchmark: Optional[PresetTrialBenchmark] = None failed: bool = False + @model_validator(mode="after") + def fail_without_benchmark(self) -> Self: + """An entry without a benchmark is one the agent could not measure, which + the prompt defines as failed; the flag is set in case the agent left it out.""" + if self.benchmark is None: + self.failed = True + return self + + +def _entry_or_failed(value: Any, handler: ValidatorFunctionWrapHandler) -> Any: + try: + return handler(value) + except ValidationError: + # A broken entry counts as a failed one, so the other entries of the + # trial are still listed. + return PresetTrialResult(failed=True) + class PresetTrial(CoreModel): """The `trials//trial.json` record as defined in the system_prompt.md. @@ -100,12 +117,18 @@ class PresetTrial(CoreModel): groups: Optional[list[list[dict[str, Any]]]] = None # Required and non-empty: `system_prompt.md` states that `trial.json` has # one `result` entry per concurrency value. - result: Annotated[list[PresetTrialResult], Field(min_length=1)] + result: Annotated[ + list[Annotated[PresetTrialResult, WrapValidator(_entry_or_failed)]], + Field(min_length=1), + ] @model_validator(mode="before") @classmethod def convert_to_result_list(cls, data: Any) -> Any: - if isinstance(data, dict) and "result" not in data and "benchmark" in data: + # An old `trial.json` has no `result` list. Its `benchmark`, + # `context_length` and `failed` are at the top level, and a failed one + # may have no `benchmark` at all. + if isinstance(data, dict) and "result" not in data: data = dict(data) data["result"] = [ { diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index a5af1de5d7..9e667b6529 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -316,35 +316,35 @@ def _format_creation_objective( return f"[secondary]{' '.join(parts)}[/]" -def _breaches_constraints(preset: AnyStoredPreset) -> bool: +def _passing_result(preset: AnyStoredPreset) -> list[PresetResult]: + """The measured entries that met the requested constraints.""" + measured = [r for r in preset.result if r.benchmark is not None] if not isinstance(preset, VerifiedPreset): # A pulled preset carries no requested constraints to breach. - return False - configuration = preset.configuration - # One breaching benchmark is enough: a service that misses its target at any - # measured concurrency is flagged. Entries without a benchmark are skipped. - for r in preset.result: - if r.benchmark is None: - continue - if ( - configuration.max_ttft is not None - and r.benchmark.metrics.ttft_ms.p50 > configuration.max_ttft - ): - return True - if ( - configuration.min_context_length is not None - and r.context_length is not None - and r.context_length < configuration.min_context_length - ): - return True - return False + return measured + return [r for r in measured if not _breaches_constraints(r, preset.configuration)] + + +def _breaches_constraints(r: PresetResult, configuration: PresetConfiguration) -> bool: + assert r.benchmark is not None + if ( + configuration.max_ttft is not None + and r.benchmark.metrics.ttft_ms.p50 > configuration.max_ttft + ): + return True + return ( + configuration.min_context_length is not None + and r.context_length is not None + and r.context_length < configuration.min_context_length + ) def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) -> str: + passing = _passing_result(preset) + if passing: + return _format_result_benchmark(passing, verbose=verbose) text = _format_result_benchmark(preset.result, verbose=verbose) - if _breaches_constraints(preset): - return f"[secondary]*{text}[/]" - return text + return f"[secondary]*{text}[/]" # Entries of a stored preset and of a session's trial share the fields read here. diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index ced935329c..a1f4c27674 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -610,7 +610,7 @@ def _summarize_session_trials(path: Path) -> PresetTrialsSummary: count=len(trials), best=best, best_failed=best_failed, - series=[_max_tok_s(trial) for trial in trials], + series=[_max_tok_s(_scored_trial(trial)) for trial in trials], failed=[_trial_failed(trial) for trial in trials], gpu=gpu, ) @@ -646,8 +646,8 @@ def _passing_trial_results(trial: PresetTrial) -> list[PresetTrialResult]: def _max_tok_s(trial: PresetTrial) -> Optional[float]: - """The trial's score for selection and its sparkline bar: the highest tok/s - over its entries that have a benchmark; None when no entry has one.""" + """The highest tok/s over the trial's entries that have a benchmark; None + when no entry has one.""" values = [ entry.benchmark.effective_output_tok_per_s for entry in trial.result @@ -658,6 +658,13 @@ def _max_tok_s(trial: PresetTrial) -> Optional[float]: return max(values) +def _scored_trial(trial: PresetTrial) -> PresetTrial: + """Returns trial with only the entries that met the constraints, or the original trial if none did.""" + if _trial_failed(trial): + return trial + return trial.model_copy(update={"result": _passing_trial_results(trial)}) + + def _select_best_and_best_failed( trials: list[PresetTrial], ) -> tuple[Optional[PresetTrial], Optional[PresetTrial]]: @@ -668,10 +675,7 @@ def _select_best_and_best_failed( best_failed: Optional[PresetTrial] = None best_failed_tok_s = 0.0 for trial in trials: - if _trial_failed(trial): - kept = trial - else: - kept = trial.model_copy(update={"result": _passing_trial_results(trial)}) + kept = _scored_trial(trial) tok_s = _max_tok_s(kept) if tok_s is None: continue diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 17eaeff205..e7467180a6 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -165,11 +165,27 @@ def _check_result_answers_request( ) -> None: if all(r.benchmark is None for r in result): raise CLIError("The agent's final report has no completed benchmark") + concurrencies = [r.concurrency for r in result] + if len(set(concurrencies)) != len(concurrencies): + raise CLIError("The agent's final report has more than one entry for the same concurrency") for r in result: - # A `null` benchmark is an entry the agent could not measure; a partial - # sweep is allowed, so there is nothing to compare for it. + if ( + configuration.concurrency is not None + and r.concurrency not in configuration.concurrency + ): + raise CLIError( + f"The agent's final benchmark concurrency of {r.concurrency} does not match the" + f" requested concurrency of {configuration.concurrency}" + ) + # The agent could not measure this concurrency. That is allowed, so + # there is nothing more to check for it. if r.benchmark is None: continue + if r.context_length is None: + raise CLIError( + f"The agent's final report has a benchmark at concurrency {r.concurrency}" + " but no context length" + ) if r.benchmark.workload.concurrency != r.concurrency: raise CLIError( f"The agent's final benchmark concurrency {r.concurrency} does not" @@ -201,14 +217,6 @@ def _check_workload_answers_request( f"The agent's final benchmark shared prefix of {workload.shared_prefix_tokens}" f" tokens does not match the requested {shared_prefix_tokens}" ) - if ( - configuration.concurrency is not None - and workload.concurrency not in configuration.concurrency - ): - raise CLIError( - f"The agent's final benchmark concurrency of {workload.concurrency} does not match the" - f" requested concurrency of {configuration.concurrency}" - ) def _normalized_benchmark( diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index 87b9434565..7728b209a0 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -1677,6 +1677,47 @@ def test_a_torn_result_copy_reads_as_still_in_flight(self, tmp_path): assert summary.count == 0 + def test_an_old_failed_trial_without_a_benchmark_is_counted(self, tmp_path): + # Before concurrency sweeps, a failed trial was written with no + # `benchmark` key and no `result` list. + trials_dir = _write_trials(tmp_path, []) + (trials_dir / "1").mkdir(parents=True) + (trials_dir / "1" / "trial.json").write_text( + json.dumps( + { + "resources": {"gpu": {"name": "A40", "memory": "48GB", "count": 1}}, + "failed": True, + } + ) + ) + + summary = _summarize_session_trials(trials_dir) + + assert summary.count == 1 + assert summary.series == [None] + assert summary.failed == [True] + assert summary.gpu == "A40:48GB:1" + + def test_a_broken_entry_does_not_drop_the_other_entries(self, tmp_path): + trial = PresetTrial( + result=[ + get_trial_result(tok_s=100.0, concurrency=48), + get_trial_result(tok_s=180.0, concurrency=96), + ] + ) + record = json.loads(trial.model_dump_json()) + record["result"].append({"concurrency": 192, "benchmark": {"metrics": {}}}) + trials_dir = _write_trials(tmp_path, []) + (trials_dir / "1").mkdir(parents=True) + (trials_dir / "1" / "trial.json").write_text(json.dumps(record)) + + summary = _summarize_session_trials(trials_dir) + + assert summary.count == 1 + assert summary.series == [180.0] + assert summary.failed == [False] + assert [r.failed for r in get_or_error(summary.best).result] == [False, False] + class TestReadLastSessionVerification: def test_newest_attempt_wins_and_a_missing_directory_is_not_verifying(self, tmp_path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index e3944f1efc..9498dc8ccd 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -274,6 +274,62 @@ def test_rejects_a_sweep_with_no_completed_benchmark(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) + def test_rejects_a_sweep_with_two_entries_at_the_same_concurrency(self, tmp_path): + run = get_running_service_run() + report = _sweep_report(run, measured=(48,)) + report = report.model_copy(update={"result": [report.result[0], *report.result]}) + + with pytest.raises(CLIError, match="more than one entry for the same concurrency"): + build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_rejects_an_unmeasured_entry_at_an_unrequested_concurrency(self, tmp_path): + run = get_running_service_run() + report = _sweep_report(run, measured=(48,)) + unrequested = report.result[2].model_copy(update={"concurrency": 500}) + report = report.model_copy(update={"result": [*report.result[:2], unrequested]}) + + with pytest.raises( + CLIError, + match=r"concurrency of 500 does not match the requested concurrency of \[48, 96, 192\]", + ): + build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_rejects_a_measured_entry_without_a_context_length(self, tmp_path): + run = get_running_service_run() + report = _sweep_report(run, measured=(48,)) + measured = report.result[0].model_copy(update={"context_length": None}) + report = report.model_copy(update={"result": [measured, *report.result[1:]]}) + + with pytest.raises(CLIError, match="benchmark at concurrency 48 but no context length"): + build_verified_preset( + run=run, + preset_configuration=_sweep_configuration(), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): run = get_running_service_run() report = _dataset_report(run, dataset="spec_bench") From 66144cae5726bb8b85acf8934850bea037c4da53 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Fri, 20 Nov 2026 11:20:37 +0100 Subject: [PATCH 11/12] Fix docs --- mkdocs/docs/concepts/presets.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index 7f78972314..a3d5638c45 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -361,9 +361,23 @@ The `CONSTRAINTS` column is what the creation was asked for, and `BENCHMARK` is When `concurrency` is a list, `CONSTRAINTS` shows it as `c=min..max`, and the `BENCHMARK` metrics become ranges over the concurrencies that met the constraints, e.g. `tps/user=min..max` and `ttft=min..max`. -The glyphs after the benchmark are one per trial: height is the trial's highest throughput across its concurrency values that met the constraints, a yellow bar is a trial that broke a constraint at every concurrency, and a red `·` is one that produced no benchmark at all. The shape shows whether a run converged or wandered. -Pass `-w` to watch in realtime, `-v` for more detail, or `--json` for complete preset objects. Filter with `--base` or `--repo`. +!!! info "Sparkline legend" + + Each bar in the sparklines represents one trial. The bar height is the trial's highest tok/s across concurrency values that met the constraints. + + | Bar color | Semantics | + | --- | --- | + | Red | A trial that broke constraints at every concurrency. | + | Green | The best trial that met the constraints. | + | Gray | A trial that met the constraints but had lower tok/s than the best trial. | + | Yellow | The best trial, if it broke constraints at every concurrency. | + + A red `·` represets a trial that produced no benchmark. + +For detailed benchmark data at each concurrency, pass `--json` to `dstack preset get`. + +To watch in realtime, use `-w` with `dstack preset`. Filter with `--base` or `--repo`. Also, use `-v` for verbose output. ### Delete presets From 818e6563f331a48f63d9c80778313187f181363f Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Fri, 20 Nov 2026 11:55:21 +0100 Subject: [PATCH 12/12] Fix harness --- .../presets/resources/system_prompt.md | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 6eb660e7c2..a010ad78bf 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -169,7 +169,7 @@ tweaking parameters further. +Problem: With single concurrency value, each trial has only one output_tok_per_s. For preset agent comparing trials with single output_tok_per_s value is straight forward, if trial1's output_tok_per_s is greater that trail2's then choose trial1. With concurrency sweep, each trial will have multiple output_tok_per_s values.This means with concurrency sweep, preset agent has to compare trials based output_tok_per_s sweeps. Currently we do not have a standard method for output_tok_per_s sweep comparision between trials and we let agent decide the method.--> Before starting trials, analyze the previous sessions' results provided in @@ -256,6 +256,8 @@ meet the constraints (see `# Constraints`). When a trial that changed several things fails, be mindful of which specific change was the root cause. +## Trial Report + `trials//trial.json` is a JSON object. 1. In case the task is not using node groups, the fields are these and no @@ -385,11 +387,16 @@ never part of the measured metrics. All verification and benchmark requests must succeed. -Record every benchmark using the following structure and field names (values -are illustrative): for a trial, in `trials//trial.json` as -`result[concurrency == ].benchmark`; for the final service, as -`final_report.json.result[concurrency == ].benchmark`, where `` is -a concurrency value from `constraints.json`. +Use the format below for the `benchmark` field of each `result` entry in: + +- `trials//trial.json`, as defined in `## Trial Report`. +- `final_report.json`, as defined in `# Final Report`. + +The benchmark's `workload.concurrency` must equal the entry's `concurrency` +value from `constraints.json`. + +The JSON below is one `benchmark` object, not the whole file. Use its exact +structure and field names; values are illustrative. ```json { @@ -419,20 +426,16 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free -invocation. For the final benchmark, use streaming responses. Set -`result[concurrency == ].benchmark.workload.concurrency` to the same value as -`result[concurrency == ].concurrency`, where `` -is a concurrency value. Produce every field of the structure, -and calculate all metrics from the -`num_requests` measured requests only — exclude setup, health-check, and -warmup requests. Never invent missing values. - -After each benchmark, find the largest context the configuration handles by -sending real requests, and record it: for a trial, in -`trials//trial.json` as `result[concurrency == ].context_length`; -for the final service, as -`final_report.json.result[concurrency == ].context_length`, where `` -is a concurrency value. Stopping at the required minimum is not enough. +invocation. Use streaming responses for the final benchmark. + +Include every field in the benchmark format. Calculate all metrics from the +`num_requests` measured requests only; exclude setup, health-check, and warmup +requests. Never invent missing values. + +After each benchmark, send real requests at the same concurrency to find the +largest context the configuration can handle. Record it in the `context_length` +field of the same `result` entry as that benchmark (see `## Trial Report` and +`# Final Report`). Do not stop at the required minimum. @@ -603,9 +606,9 @@ print, copy, or summarize tokens, secrets, or environment variable values. Do not expose the value of `DSTACK_TOKEN` or the value of any environment variable listed under `env` in `constraints.json`. -Do not put secret values in `final_report.json`, or print them. Use env -references in `final_report.json.service_yaml`; use environment variable -names or redacted values in `final_report.json.result[concurrency == ].benchmark.command`, where ``is a concurrency value. +Do not put secret values in `final_report.json` or print them. Use env +references in `service_yaml` and environment variable names or redacted values +in each `result` entry's `benchmark.command` field (see `# Final Report`). # Final Report @@ -622,7 +625,7 @@ trial remained), include exactly: - `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `result`: list of final service result objects, one object for each value of `concurrency` from `constraints.json`. Each object has fields no other than below: +- `result`: list of final service result objects, one object for each value of `concurrency` from `constraints.json`. Each object has only the fields below: - `concurrency`: the concurrency value for this result - `context_length`: the largest context verified for the final service at this concurrency, as described in `## Benchmark`. `null` only when the benchmark couldn't be done at this concurrency.