diff --git a/mkdocs/docs/concepts/metrics.md b/mkdocs/docs/concepts/metrics.md
index bf91901db0..1b6bfd0f66 100644
--- a/mkdocs/docs/concepts/metrics.md
+++ b/mkdocs/docs/concepts/metrics.md
@@ -20,6 +20,7 @@ This tab displays key CPU, memory, and GPU metrics collected during the last hou
As an alternative to the UI, you can track essential metrics via the CLI.
The `dstack metrics` command charts CPU, memory, and GPU utilization over the last hour.
+For jobs on NVIDIA GPUs, it adds a `POWER` column that charts power draw against each GPU's enforced power limit.
@@ -126,6 +127,7 @@ telemetry, and more.
| `dstack_job_gpu_usage_ratio` | *gauge* | Job GPU usage, percent (as 0.0-1.0) | `0.93` |
| `dstack_job_gpu_memory_total_bytes` | *gauge* | Total GPU memory allocated for the job, bytes | `8589934592.0` |
| `dstack_job_gpu_memory_usage_bytes` | *gauge* | GPU memory used by the job, bytes | `1048576.0` |
+ | `dstack_job_gpu_power_draw_watts` | *gauge* | Power draw of the job's GPU, watts (NVIDIA only) | `285.43` |
| `DCGM_FI_DEV_GPU_UTIL` | *gauge* | GPU utilization (in %) | |
| `DCGM_FI_DEV_MEM_COPY_UTIL` | *gauge* | Memory utilization (in %) | |
| `DCGM_FI_DEV_ENC_UTIL` | *gauge* | Encoder utilization (in %) | |
diff --git a/runner/internal/runner/metrics/metrics.go b/runner/internal/runner/metrics/metrics.go
index 34f22c731a..688d91dce3 100644
--- a/runner/internal/runner/metrics/metrics.go
+++ b/runner/internal/runner/metrics/metrics.go
@@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
+ "math"
"os"
"os/exec"
"path"
@@ -159,13 +160,13 @@ func (s *MetricsCollector) GetGPUMetrics(ctx context.Context) ([]schemas.GPUMetr
}
func (s *MetricsCollector) GetNVIDIAGPUMetrics(ctx context.Context) ([]schemas.GPUMetrics, error) {
- cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=memory.used,utilization.gpu", "--format=csv,noheader,nounits")
+ cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=memory.used,utilization.gpu,power.draw,enforced.power.limit", "--format=csv,noheader,nounits")
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
return []schemas.GPUMetrics{}, fmt.Errorf("failed to execute nvidia-smi: %w", err)
}
- return parseNVIDIASMILikeMetrics(out.String())
+ return parseNVIDIASMILikeMetrics(out.String(), true)
}
func (s *MetricsCollector) GetAMDGPUMetrics(ctx context.Context) ([]schemas.GPUMetrics, error) {
@@ -241,16 +242,22 @@ func (s *MetricsCollector) GetIntelAcceleratorMetrics(ctx context.Context) ([]sc
if err := cmd.Run(); err != nil {
return []schemas.GPUMetrics{}, fmt.Errorf("failed to execute hl-smi: %w", err)
}
- return parseNVIDIASMILikeMetrics(out.String())
+ return parseNVIDIASMILikeMetrics(out.String(), false)
}
-func parseNVIDIASMILikeMetrics(output string) ([]schemas.GPUMetrics, error) {
+// parseNVIDIASMILikeMetrics parses memory used and utilization, followed by power draw and
+// limit if withPower is set (nvidia-smi is queried for them, hl-smi is not).
+func parseNVIDIASMILikeMetrics(output string, withPower bool) ([]schemas.GPUMetrics, error) {
metrics := []schemas.GPUMetrics{}
+ columns := 2
+ if withPower {
+ columns = 4
+ }
lines := strings.Split(strings.TrimSpace(output), "\n")
for _, line := range lines {
parts := strings.Split(line, ", ")
- if len(parts) != 2 {
+ if len(parts) != columns {
continue
}
memUsed, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
@@ -261,11 +268,27 @@ func parseNVIDIASMILikeMetrics(output string) ([]schemas.GPUMetrics, error) {
if err != nil {
return metrics, fmt.Errorf("failed to parse accelerator utilization: %w", err)
}
- metrics = append(metrics, schemas.GPUMetrics{
+ gpuMetrics := schemas.GPUMetrics{
GPUMemoryUsage: memUsed * 1024 * 1024, // Convert MiB to bytes
GPUUtil: utilization,
- })
+ }
+ if withPower {
+ gpuMetrics.GPUPowerDraw = parseWatts(parts[2])
+ gpuMetrics.GPUPowerLimit = parseWatts(parts[3])
+ }
+ metrics = append(metrics, gpuMetrics)
}
return metrics, nil
}
+
+// parseWatts returns nil for readings the GPU does not provide, which nvidia-smi prints as
+// [N/A], [Not Supported] or [Unknown Error], so that they do not drop the other metrics.
+// NaN and infinity are dropped too, as they cannot be encoded in the JSON response.
+func parseWatts(value string) *float64 {
+ watts, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
+ if err != nil || math.IsNaN(watts) || math.IsInf(watts, 0) {
+ return nil
+ }
+ return &watts
+}
diff --git a/runner/internal/runner/metrics/metrics_test.go b/runner/internal/runner/metrics/metrics_test.go
index 844b02bd30..f1cf1bb737 100644
--- a/runner/internal/runner/metrics/metrics_test.go
+++ b/runner/internal/runner/metrics/metrics_test.go
@@ -1,6 +1,7 @@
package metrics
import (
+ "encoding/json"
"testing"
"github.com/dstackai/dstack/runner/internal/runner/schemas"
@@ -45,3 +46,56 @@ func TestGetAMDGPUMetrics_ErrorGPUUtilNA(t *testing.T) {
assert.ErrorContains(t, err, "GPU utilization is N/A")
assert.Nil(t, metrics)
}
+
+func TestParseNVIDIASMILikeMetrics(t *testing.T) {
+ watts := func(w float64) *float64 { return &w }
+
+ cases := []struct {
+ name string
+ output string
+ withPower bool
+ expected []schemas.GPUMetrics
+ }{
+ {
+ // NVIDIA driver 620.70.06, Tesla T4 under load
+ name: "nvidia-smi",
+ output: "525, 100, 68.23, 70.00\n",
+ withPower: true,
+ expected: []schemas.GPUMetrics{
+ {GPUMemoryUsage: 550502400, GPUUtil: 100, GPUPowerDraw: watts(68.23), GPUPowerLimit: watts(70)},
+ },
+ },
+ {
+ // the T4 at idle, then made-up lines for GPUs without power readings
+ name: "nvidia-smi, GPUs without power readings",
+ output: "525, 0, 26.88, 70.00\n1024, 5, [N/A], [N/A]\n2048, 7, [Not Supported], [Not Supported]\n" +
+ "512, 9, [Unknown Error], 70.00\n256, 1, NaN, Inf\n",
+ withPower: true,
+ expected: []schemas.GPUMetrics{
+ {GPUMemoryUsage: 550502400, GPUUtil: 0, GPUPowerDraw: watts(26.88), GPUPowerLimit: watts(70)},
+ {GPUMemoryUsage: 1073741824, GPUUtil: 5},
+ {GPUMemoryUsage: 2147483648, GPUUtil: 7},
+ {GPUMemoryUsage: 536870912, GPUUtil: 9, GPUPowerLimit: watts(70)},
+ {GPUMemoryUsage: 268435456, GPUUtil: 1},
+ },
+ },
+ {
+ name: "hl-smi",
+ output: "512, 3\n",
+ expected: []schemas.GPUMetrics{
+ {GPUMemoryUsage: 536870912, GPUUtil: 3},
+ },
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ metrics, err := parseNVIDIASMILikeMetrics(tc.output, tc.withPower)
+ assert.NoError(t, err)
+ assert.Equal(t, tc.expected, metrics)
+ // a value json.Marshal rejects would fail the whole metrics response
+ _, err = json.Marshal(metrics)
+ assert.NoError(t, err)
+ })
+ }
+}
diff --git a/runner/internal/runner/schemas/schemas.go b/runner/internal/runner/schemas/schemas.go
index c9102d732d..346c0b5981 100644
--- a/runner/internal/runner/schemas/schemas.go
+++ b/runner/internal/runner/schemas/schemas.go
@@ -148,8 +148,10 @@ type HealthcheckResponse struct {
}
type GPUMetrics struct {
- GPUMemoryUsage uint64 `json:"gpu_memory_usage_bytes"`
- GPUUtil uint64 `json:"gpu_util_percent"`
+ GPUMemoryUsage uint64 `json:"gpu_memory_usage_bytes"`
+ GPUUtil uint64 `json:"gpu_util_percent"`
+ GPUPowerDraw *float64 `json:"gpu_power_draw_watts,omitempty"`
+ GPUPowerLimit *float64 `json:"gpu_power_limit_watts,omitempty"`
}
type SystemMetrics struct {
diff --git a/src/dstack/_internal/cli/utils/metrics.py b/src/dstack/_internal/cli/utils/metrics.py
index f921d18f02..f98fba5141 100644
--- a/src/dstack/_internal/cli/utils/metrics.py
+++ b/src/dstack/_internal/cli/utils/metrics.py
@@ -28,17 +28,28 @@
NOW = "now"
LIVE_THRESHOLD = timedelta(seconds=3 * WATCH_INTERVAL_SECONDS)
-_FIXED_COLUMNS = 30
+_FIXED_COLUMNS = 32
"""Everything but the sparklines and the job label: the `gpu=N` column, both numbers, and
-the table's padding. Hand-measured against a `589GB/1480GB`-sized number; a wider one
-overflows and Rich ellipsizes the labels rather than shrinking the sparklines."""
+the table's padding. Measured against `100%` and a `9999GB/9999GB`-sized number, which fits
+hosts under 10 TB; a wider one overflows and Rich ellipsizes the labels rather than shrinking
+the sparklines."""
+
+_POWER_FIXED_COLUMNS = 14
+"""What the power column adds besides its sparkline: a `1000W/1000W`-sized number, the space
+before it, and the column's padding."""
_SPARKLINE_COLUMNS = 2
-def _spark_width(console_width: int, label_width: int = 0) -> int:
- budget = console_width - _FIXED_COLUMNS - label_width
- return max(MIN_SPARK_WIDTH, min(MAX_SPARK_WIDTH, budget // _SPARKLINE_COLUMNS))
+def _spark_budget(console_width: int, label_width: int, show_power: bool) -> int:
+ fixed = _FIXED_COLUMNS + (_POWER_FIXED_COLUMNS if show_power else 0)
+ columns = _SPARKLINE_COLUMNS + (1 if show_power else 0)
+ return (console_width - fixed - label_width) // columns
+
+
+def _spark_width(console_width: int, label_width: int = 0, show_power: bool = False) -> int:
+ budget = _spark_budget(console_width, label_width, show_power)
+ return max(MIN_SPARK_WIDTH, min(MAX_SPARK_WIDTH, budget))
def get_metrics_table(
@@ -48,7 +59,13 @@ def get_metrics_table(
) -> RenderableType:
labels = job_labels(jobs)
label_width = max((len(label) for label in labels), default=0)
- width = _spark_width(console_width or console.width, label_width)
+ console_width = console_width or console.width
+ # Older runners and non-NVIDIA GPUs report no power. The column is also left out if three
+ # charts of the minimum width don't fit, as Rich would cut the labels to make room.
+ show_power = any(_reports_power(m) for m in metrics) and (
+ _spark_budget(console_width, label_width, show_power=True) >= MIN_SPARK_WIDTH
+ )
+ width = _spark_width(console_width, label_width, show_power)
window = _shared_window(metrics)
table = Table(box=None)
@@ -57,18 +74,21 @@ def get_metrics_table(
table.add_column("", style="secondary", no_wrap=True)
table.add_column("UTILIZATION", no_wrap=True)
table.add_column("MEMORY", no_wrap=True)
+ if show_power:
+ table.add_column("POWER", no_wrap=True)
+ empty_row = [""] * len(table.columns)
for index, (job, job_metrics) in enumerate(zip(jobs, metrics)):
if index:
- table.add_row("", "", "", "")
- _add_job(table, job, job_metrics, width, labels[index], window)
+ table.add_row(*empty_row)
+ _add_job(table, job, job_metrics, width, labels[index], window, show_power)
if window is not None:
- table.add_row("", "", "", "")
+ table.add_row(*empty_row)
# the axis spans the widest chart drawn: a job with fewer samples than cells draws
# one cell per sample and cannot fill its share
axis = _time_axis(max(_cells_drawn(m, window, width) for m in metrics), *window)
- table.add_row("", "", axis, axis)
+ table.add_row("", "", *[axis] * (len(table.columns) - 2))
return table
@@ -100,23 +120,39 @@ def _add_job(
width: int,
label: str,
window: Optional[tuple[datetime, datetime]],
+ show_power: bool,
) -> None:
resources = _get_resources(job)
blanks = _blank_cells(metrics, window, width)
cells = width - blanks
- table.add_row(
+ host_row: List[RenderableType] = [
label,
"cpu",
_pad(_cpu_cell(metrics, resources, cells), blanks),
_pad(_memory_cell(metrics, resources, cells), blanks),
- )
+ ]
+ if show_power:
+ host_row.append("")
+ table.add_row(*host_row)
for index in range(_gpus_num(metrics, resources)):
- table.add_row(
+ gpu_row: List[RenderableType] = [
"",
f"gpu={index}",
_pad(_gpu_util_cell(metrics, index, cells), blanks),
_pad(_gpu_memory_cell(metrics, resources, index, cells), blanks),
- )
+ ]
+ if show_power:
+ power_blanks = max(blanks, _blank_cells(_power_series(metrics, index), window, width))
+ gpu_row.append(
+ _pad(_gpu_power_cell(metrics, index, width - power_blanks), power_blanks)
+ )
+ table.add_row(*gpu_row)
+
+
+def _power_series(job_metrics: JobMetrics, index: int) -> JobMetrics:
+ """Power draw of GPU `index` only. It may start later than the other series."""
+ name = f"gpu_power_draw_watts_gpu{index}"
+ return JobMetrics(metrics=[m for m in job_metrics.metrics if m.name == name])
def _shared_window(metrics: Sequence[JobMetrics]) -> Optional[tuple[datetime, datetime]]:
@@ -197,6 +233,20 @@ def _gpu_util_cell(job_metrics: JobMetrics, index: int, width: int) -> Text:
return Text.assemble(sparkline(values, width, GPU_RAMP), " ", f"{values[-1]:.0f}%")
+def _gpu_power_cell(job_metrics: JobMetrics, index: int, width: int) -> Text:
+ values = _metric_values(job_metrics, f"gpu_power_draw_watts_gpu{index}")
+ if not values:
+ return no_data()
+ limit = _latest_value(job_metrics, f"gpu_power_limit_watts_gpu{index}")
+ if not limit:
+ # without the limit there is no fixed scale to chart against
+ return Text(f"{values[-1]:.0f}W")
+ percents = [v / limit * 100 for v in values]
+ return Text.assemble(
+ sparkline(percents, width, GPU_RAMP), " ", f"{values[-1]:.0f}W/{limit:.0f}W"
+ )
+
+
def _capacity_cell(values: List[float], total: Optional[float], width: int, ramp: Ramp) -> Text:
"""A memory chart drawn against capacity, labelled `used/total`."""
percents = [v / total * 100 for v in values] if total else values
@@ -252,6 +302,10 @@ def _latest_value(job_metrics: JobMetrics, name: str) -> Optional[Any]:
return values[-1] if values else None
+def _reports_power(job_metrics: JobMetrics) -> bool:
+ return any(m.name.startswith("gpu_power_draw_watts_gpu") for m in job_metrics.metrics)
+
+
def _gpus_num(job_metrics: JobMetrics, resources: Optional[Resources]) -> int:
if resources is not None and resources.gpus:
return len(resources.gpus)
diff --git a/src/dstack/_internal/server/background/scheduled_tasks/metrics.py b/src/dstack/_internal/server/background/scheduled_tasks/metrics.py
index 103e7795e8..a4e7e9b119 100644
--- a/src/dstack/_internal/server/background/scheduled_tasks/metrics.py
+++ b/src/dstack/_internal/server/background/scheduled_tasks/metrics.py
@@ -152,6 +152,8 @@ async def _collect_job_metrics(job_model: JobModel) -> Optional[JobMetricsPoint]
gpus_memory_usage_bytes = [g.gpu_memory_usage_bytes for g in res.gpus]
gpus_util_percent = [g.gpu_util_percent for g in res.gpus]
+ gpus_power_draw_watts = [g.gpu_power_draw_watts for g in res.gpus]
+ gpus_power_limit_watts = [g.gpu_power_limit_watts for g in res.gpus]
return JobMetricsPoint(
job_id=job_model.id,
@@ -161,9 +163,18 @@ async def _collect_job_metrics(job_model: JobModel) -> Optional[JobMetricsPoint]
memory_working_set_bytes=res.memory_working_set_bytes,
gpus_memory_usage_bytes=json.dumps(gpus_memory_usage_bytes),
gpus_util_percent=json.dumps(gpus_util_percent),
+ gpus_power_draw_watts=_dump_gpus_power(gpus_power_draw_watts),
+ gpus_power_limit_watts=_dump_gpus_power(gpus_power_limit_watts),
)
+def _dump_gpus_power(values: List[Optional[float]]) -> Optional[str]:
+ # NULL if no GPU has a reading, as with older runners and non-NVIDIA GPUs
+ if all(v is None for v in values):
+ return None
+ return json.dumps(values)
+
+
@runner_ssh_tunnel
def _pull_runner_metrics(
addresses: Mapping[int, client.LocalAddress],
diff --git a/src/dstack/_internal/server/migrations/versions/2026/11_20_0019_eb9ea2fdd147_add_jobmetricspoint_gpu_power_columns.py b/src/dstack/_internal/server/migrations/versions/2026/11_20_0019_eb9ea2fdd147_add_jobmetricspoint_gpu_power_columns.py
new file mode 100644
index 0000000000..45092e2fdf
--- /dev/null
+++ b/src/dstack/_internal/server/migrations/versions/2026/11_20_0019_eb9ea2fdd147_add_jobmetricspoint_gpu_power_columns.py
@@ -0,0 +1,34 @@
+"""Add JobMetricsPoint GPU power columns
+
+Revision ID: eb9ea2fdd147
+Revises: 2d2692d5b3c2
+Create Date: 2026-11-20 00:19:27.385130+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "eb9ea2fdd147"
+down_revision = "2d2692d5b3c2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("job_metrics_points", schema=None) as batch_op:
+ batch_op.add_column(sa.Column("gpus_power_draw_watts", sa.Text(), nullable=True))
+ batch_op.add_column(sa.Column("gpus_power_limit_watts", sa.Text(), nullable=True))
+
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("job_metrics_points", schema=None) as batch_op:
+ batch_op.drop_column("gpus_power_limit_watts")
+ batch_op.drop_column("gpus_power_draw_watts")
+
+ # ### end Alembic commands ###
diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py
index 27c5311ea1..85a1918a9b 100644
--- a/src/dstack/_internal/server/models.py
+++ b/src/dstack/_internal/server/models.py
@@ -1240,6 +1240,13 @@ class JobMetricsPoint(BaseModel):
"""`gpus_memory_usage_bytes` stores a JSON-encoded list of metric values with length `len(gpus)`."""
gpus_util_percent: Mapped[str] = mapped_column(Text)
"""`gpus_util_percent` stores a JSON-encoded list of metric values with length `len(gpus)`."""
+ gpus_power_draw_watts: Mapped[Optional[str]] = mapped_column(Text)
+ """`gpus_power_draw_watts` stores a JSON-encoded list of metric values with length `len(gpus)`,
+ with `null` for GPUs without a reading. `None` if no GPU reported power, or if the point was
+ written by a server without this column.
+ """
+ gpus_power_limit_watts: Mapped[Optional[str]] = mapped_column(Text)
+ """`gpus_power_limit_watts` has the same format as `gpus_power_draw_watts`."""
class JobPrometheusMetrics(BaseModel):
diff --git a/src/dstack/_internal/server/routers/metrics.py b/src/dstack/_internal/server/routers/metrics.py
index b44d4a86a3..feb2010c6b 100644
--- a/src/dstack/_internal/server/routers/metrics.py
+++ b/src/dstack/_internal/server/routers/metrics.py
@@ -58,6 +58,8 @@ async def get_job_metrics(
* `gpu_memory_total_bytes`
* `gpu_memory_usage_bytes_gpu{i}`
* `gpu_util_percent_gpu{i}`
+ * `gpu_power_draw_watts_gpu{i}` (NVIDIA only; may have fewer samples than other metrics)
+ * `gpu_power_limit_watts_gpu{i}` (NVIDIA only; may have fewer samples than other metrics)
"""
_, project = user_project
diff --git a/src/dstack/_internal/server/schemas/runner.py b/src/dstack/_internal/server/schemas/runner.py
index 3a3aac4a0a..aa4ff51e05 100644
--- a/src/dstack/_internal/server/schemas/runner.py
+++ b/src/dstack/_internal/server/schemas/runner.py
@@ -182,6 +182,10 @@ class ComponentInstallRequest(CoreModel):
class GPUMetrics(CoreModel):
gpu_memory_usage_bytes: int
gpu_util_percent: int
+ gpu_power_draw_watts: Optional[float] = None
+ """`gpu_power_draw_watts` is not set by old runners, non-NVIDIA GPUs, and GPUs without power readings."""
+ gpu_power_limit_watts: Optional[float] = None
+ """`gpu_power_limit_watts` is the enforced limit. Set under the same conditions as `gpu_power_draw_watts`."""
class MetricsResponse(CoreModel):
diff --git a/src/dstack/_internal/server/services/metrics.py b/src/dstack/_internal/server/services/metrics.py
index d8f3c96830..7dba4a1d99 100644
--- a/src/dstack/_internal/server/services/metrics.py
+++ b/src/dstack/_internal/server/services/metrics.py
@@ -53,6 +53,13 @@ async def get_job_metrics(
return _calculate_job_metrics(job_model, points)
+def load_gpus_power(value: Optional[str], gpus_num: int) -> list[Optional[float]]:
+ """Decodes `JobMetricsPoint.gpus_power_draw_watts` or `gpus_power_limit_watts`."""
+ if value is None:
+ return [None] * gpus_num
+ return json.loads(value)
+
+
def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint]) -> JobMetrics:
timestamps: list[datetime] = []
cpu_usage_points: list[int] = []
@@ -60,6 +67,8 @@ def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint
memory_working_set_points: list[int] = []
gpus_memory_usage_points: defaultdict[int, list[int]] = defaultdict(list)
gpus_util_points: defaultdict[int, list[int]] = defaultdict(list)
+ gpus_power_draw_points: defaultdict[int, list[tuple[datetime, float]]] = defaultdict(list)
+ gpus_power_limit_points: defaultdict[int, list[tuple[datetime, float]]] = defaultdict(list)
cpus_detected_num: Optional[int] = None
memory_total: Optional[int] = None
@@ -81,7 +90,8 @@ def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint
gpus_detected_num: Optional[int] = None
gpus_detected_num_mismatch: bool = False
for point, prev_point in zip(points, points[1:]):
- timestamps.append(_unix_time_micro_to_datetime(point.timestamp_micro))
+ timestamp = _unix_time_micro_to_datetime(point.timestamp_micro)
+ timestamps.append(timestamp)
cpu_usage_points.append(_get_cpu_usage(point, prev_point))
memory_usage_points.append(point.memory_usage_bytes)
memory_working_set_points.append(point.memory_working_set_bytes)
@@ -92,9 +102,16 @@ def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint
if len(gpus_memory_usage) != gpus_detected_num or len(gpus_util) != gpus_detected_num:
gpus_detected_num_mismatch = True
if not gpus_detected_num_mismatch:
+ gpus_power_draw = load_gpus_power(point.gpus_power_draw_watts, gpus_detected_num)
+ gpus_power_limit = load_gpus_power(point.gpus_power_limit_watts, gpus_detected_num)
for i in range(gpus_detected_num):
gpus_memory_usage_points[i].append(gpus_memory_usage[i])
gpus_util_points[i].append(gpus_util[i])
+ power_draw, power_limit = gpus_power_draw[i], gpus_power_limit[i]
+ if power_draw is not None:
+ gpus_power_draw_points[i].append((timestamp, power_draw))
+ if power_limit is not None:
+ gpus_power_limit_points[i].append((timestamp, power_limit))
metrics: list[Metric] = [
Metric(
@@ -146,6 +163,14 @@ def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint
values=gpu_util_points,
)
)
+ for index, gpu_power_draw_points in gpus_power_draw_points.items():
+ metrics.append(
+ _make_sparse_metric(f"gpu_power_draw_watts_gpu{index}", gpu_power_draw_points)
+ )
+ for index, gpu_power_limit_points in gpus_power_limit_points.items():
+ metrics.append(
+ _make_sparse_metric(f"gpu_power_limit_watts_gpu{index}", gpu_power_limit_points)
+ )
return JobMetrics(metrics=metrics)
@@ -157,6 +182,15 @@ def _make_constant_metric(name: str, timestamps: list[datetime], value: float) -
)
+def _make_sparse_metric(name: str, points: list[tuple[datetime, float]]) -> Metric:
+ """A metric with its own timestamps, as it skips points that have no value."""
+ return Metric(
+ name=name,
+ timestamps=[timestamp for timestamp, _ in points],
+ values=[value for _, value in points],
+ )
+
+
def _get_cpu_usage(last_point: JobMetricsPoint, prev_point: JobMetricsPoint) -> int:
window = last_point.timestamp_micro - prev_point.timestamp_micro
if window == 0:
diff --git a/src/dstack/_internal/server/services/prometheus/custom_metrics.py b/src/dstack/_internal/server/services/prometheus/custom_metrics.py
index 1f0015c299..ba7435ad83 100644
--- a/src/dstack/_internal/server/services/prometheus/custom_metrics.py
+++ b/src/dstack/_internal/server/services/prometheus/custom_metrics.py
@@ -26,6 +26,7 @@
)
from dstack._internal.server.services.instances import get_instance_offer
from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.server.services.metrics import load_gpus_power
from dstack._internal.server.services.runs import get_run_spec
from dstack._internal.utils.common import get_current_datetime
@@ -188,10 +189,13 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
metrics.add_sample(_JOB_MEMORY_WORKING_SET, labels, jmp.memory_working_set_bytes)
if gpus:
gpu_memory_total = gpus[0].memory_mib * 1024 * 1024
- for gpu_num, (gpu_util, gpu_memory_usage) in enumerate(
+ gpus_util = json.loads(jmp.gpus_util_percent)
+ gpus_power_draw = load_gpus_power(jmp.gpus_power_draw_watts, len(gpus_util))
+ for gpu_num, (gpu_util, gpu_memory_usage, gpu_power_draw) in enumerate(
zip(
- json.loads(jmp.gpus_util_percent),
+ gpus_util,
json.loads(jmp.gpus_memory_usage_bytes),
+ gpus_power_draw,
)
):
gpu_labels = labels.copy()
@@ -199,6 +203,8 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
metrics.add_sample(_JOB_GPU_USAGE_RATIO, gpu_labels, gpu_util / 100)
metrics.add_sample(_JOB_GPU_MEMORY_TOTAL, gpu_labels, gpu_memory_total)
metrics.add_sample(_JOB_GPU_MEMORY_USAGE, gpu_labels, gpu_memory_usage)
+ if gpu_power_draw is not None:
+ metrics.add_sample(_JOB_GPU_POWER_DRAW, gpu_labels, gpu_power_draw)
jpm = job_prometheus_metrics.get(job.id)
if jpm is not None:
for metric in text_string_to_metric_families(jpm.text):
@@ -227,6 +233,7 @@ async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
_JOB_GPU_USAGE_RATIO = "dstack_job_gpu_usage_ratio"
_JOB_GPU_MEMORY_TOTAL = "dstack_job_gpu_memory_total_bytes"
_JOB_GPU_MEMORY_USAGE = "dstack_job_gpu_memory_usage_bytes"
+_JOB_GPU_POWER_DRAW = "dstack_job_gpu_power_draw_watts"
class _Metrics(dict[str, Metric]):
@@ -287,6 +294,7 @@ class _JobMetrics(_Metrics):
(_JOB_GPU_USAGE_RATIO, _GAUGE, "Job GPU usage, percent (as 0.0-1.0)"),
(_JOB_GPU_MEMORY_TOTAL, _GAUGE, "Total GPU memory allocated for the job, bytes"),
(_JOB_GPU_MEMORY_USAGE, _GAUGE, "GPU memory used by the job, bytes"),
+ (_JOB_GPU_POWER_DRAW, _GAUGE, "Power draw of the job's GPU, watts (NVIDIA only)"),
]
diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py
index 73b27b70a1..202904ffd6 100644
--- a/src/dstack/_internal/server/testing/common.py
+++ b/src/dstack/_internal/server/testing/common.py
@@ -1317,7 +1317,10 @@ async def create_job_metrics_point(
memory_working_set_bytes: int = 1024,
gpus_memory_usage_bytes: Optional[List[int]] = None,
gpus_util_percent: Optional[List[int]] = None,
+ gpus_power_draw_watts: Optional[List[Optional[float]]] = None,
+ gpus_power_limit_watts: Optional[List[Optional[float]]] = None,
) -> JobMetricsPoint:
+ """Unset power lists are stored as NULL, as for a point without power readings."""
timestamp_micro = int(timestamp.timestamp() * 1_000_000)
if gpus_memory_usage_bytes is None:
gpus_memory_usage_bytes = []
@@ -1331,6 +1334,12 @@ async def create_job_metrics_point(
memory_working_set_bytes=memory_working_set_bytes,
gpus_memory_usage_bytes=json.dumps(gpus_memory_usage_bytes),
gpus_util_percent=json.dumps(gpus_util_percent),
+ gpus_power_draw_watts=(
+ json.dumps(gpus_power_draw_watts) if gpus_power_draw_watts is not None else None
+ ),
+ gpus_power_limit_watts=(
+ json.dumps(gpus_power_limit_watts) if gpus_power_limit_watts is not None else None
+ ),
)
session.add(jmp)
await session.commit()
diff --git a/src/tests/_internal/cli/utils/test_metrics.py b/src/tests/_internal/cli/utils/test_metrics.py
index 099b86238f..ee643b84c8 100644
--- a/src/tests/_internal/cli/utils/test_metrics.py
+++ b/src/tests/_internal/cli/utils/test_metrics.py
@@ -1,6 +1,6 @@
import re
from datetime import datetime, timedelta, timezone
-from typing import List, Tuple
+from typing import List, Optional, Tuple
from unittest.mock import MagicMock
import pytest
@@ -41,6 +41,7 @@ def make_run(
replica: int = 0,
job_num: int = 0,
group: str = "default",
+ memory_gb: int = 32,
) -> Tuple[MagicMock, JobMetrics]:
"""A job and its metrics. `state` decides whether the newest sample reads as `now`."""
newest = datetime.now(timezone.utc)
@@ -62,7 +63,7 @@ def series(fn) -> List[float]:
Metric(
name="memory_working_set_bytes",
timestamps=timestamps,
- values=series(lambda t: memory(t) * 32 * GIB),
+ values=series(lambda t: memory(t) * memory_gb * GIB),
),
]
for index in range(gpus):
@@ -82,7 +83,7 @@ def series(fn) -> List[float]:
job.job_spec.replica_group = group
submission = MagicMock()
resources = MagicMock()
- resources.cpus, resources.memory_mib = cpus, 32 * 1024
+ resources.cpus, resources.memory_mib = cpus, memory_gb * 1024
resources.gpus = [MagicMock(memory_mib=CAPACITY_GB * 1024) for _ in range(gpus)]
submission.job_runtime_data.offer.instance.resources = resources
job.job_submissions = [submission]
@@ -113,9 +114,44 @@ def row(output: str, label: str) -> str:
return next(line for line in lines(output) if line.strip().startswith(label))
-def bars(line: str) -> List[int]:
- """Glyph heights of the first sparkline in `line`, left to right."""
- return [SPARKS.index(glyph) for glyph in re.findall(rf"[{SPARKS}]+", line)[0]]
+def bars(line: str, chart: int = 0) -> List[int]:
+ """Glyph heights of a sparkline in `line`, the first by default, left to right."""
+ return [SPARKS.index(glyph) for glyph in re.findall(rf"[{SPARKS}]+", line)[chart]]
+
+
+def chart_start(line: str, chart: int = 0) -> int:
+ """The column where a sparkline in `line` begins."""
+ return list(re.finditer(rf"[{SPARKS}]+", line))[chart].start()
+
+
+def fits(output: str, width: int) -> bool:
+ """Nothing wider than the terminal and no label cut short."""
+ return "…" not in output and max(len(line) for line in output.splitlines()) <= width
+
+
+def with_power(
+ metrics: JobMetrics, draw: float, limit: Optional[float] = 700.0, gpus: int = 1
+) -> JobMetrics:
+ """`metrics` plus a steady power draw on each GPU, as an NVIDIA runner reports it."""
+ timestamps = metrics.metrics[0].timestamps
+ power = []
+ for index in range(gpus):
+ power.append(
+ Metric(
+ name=f"gpu_power_draw_watts_gpu{index}",
+ timestamps=timestamps,
+ values=[draw] * len(timestamps),
+ )
+ )
+ if limit is not None:
+ power.append(
+ Metric(
+ name=f"gpu_power_limit_watts_gpu{index}",
+ timestamps=timestamps,
+ values=[limit] * len(timestamps),
+ )
+ )
+ return JobMetrics(metrics=metrics.metrics + power)
def colours(output: str, label: str) -> set:
@@ -168,6 +204,16 @@ def test_no_data_is_not_zero(self):
assert sum(1 for line in lines(missing) if "gpu=" in line) == 2
assert "1%" in render(*make_run("idle", gpus=2))
+ def test_labels_fit_on_a_large_host(self):
+ """`100%` next to `1024GB/2048GB` is as wide as the numbers get on a host under 10 TB.
+ Below 57 columns, two charts of the minimum width don't fit anyway."""
+ job, metrics = make_run("ramp", gpus=2, memory_gb=2048)
+ host = row(render(job, metrics), "job=0")
+ assert " 100% " in host and host.endswith(" 1024GB/2048GB")
+ # a step of 5 hits every remainder of the width split between charts
+ for width in range(57, 241, 5):
+ assert fits(render(job, metrics, width=width), width), width
+
class TestWindow:
@pytest.mark.parametrize("state", ["running", "terminated"])
@@ -245,6 +291,63 @@ def test_a_late_job_starts_where_it_started(self):
assert first["replica=1"] > first["replica=0"] # pushed right by the blank
+class TestPower:
+ def test_no_column_without_power(self):
+ """Older runners and non-NVIDIA GPUs report no power: the table keeps its two columns
+ rather than gaining one that reads `no data` on every row."""
+ assert "POWER" not in render(*make_run())
+
+ def test_height_is_a_fraction_of_the_limit(self):
+ """As memory against capacity: 350W on a 700W card is a half-height bar."""
+ job, metrics = make_run()
+ gpu = row(render(job, with_power(metrics, 350.0)), "gpu=0")
+ assert gpu.endswith("350W/700W")
+ assert set(bars(gpu, chart=2)) == {(len(SPARKS) - 1) // 2}
+
+ def test_without_the_limit_only_the_number_is_shown(self):
+ """With nothing to scale against, a chart would have to rescale to its own peak."""
+ job, metrics = make_run()
+ gpu = row(render(job, with_power(metrics, 350.0, limit=None)), "gpu=0")
+ assert gpu.endswith("350W")
+ assert len(re.findall(rf"[{SPARKS}]+", gpu)) == 2
+
+ def test_a_job_without_power_reads_no_data(self):
+ nvidia, other = make_run(job_num=0), make_run(job_num=1)
+ output = render([nvidia[0], other[0]], [with_power(nvidia[1], 350.0), other[1]])
+ gpus = [line for line in lines(output) if "gpu=0" in line]
+ assert gpus[0].endswith("350W/700W")
+ assert gpus[1].endswith("no data")
+
+ def test_late_power_starts_where_it_started(self):
+ """Points collected before the server stored power have none, so the power chart
+ starts halfway through the window when only the newest half of the points has it."""
+ job, metrics = make_run(samples=360)
+ metrics = with_power(metrics, 350.0)
+ late_power = JobMetrics(
+ metrics=[
+ Metric(name=m.name, timestamps=m.timestamps[:180], values=m.values[:180])
+ if m.name.startswith("gpu_power")
+ else m
+ for m in metrics.metrics
+ ]
+ )
+ output = render(job, late_power)
+ header, gpu = lines(output)[0], row(output, "gpu=0")
+ width = len(bars(gpu, chart=0))
+ assert chart_start(gpu, chart=0) == header.index("UTILIZATION")
+ assert abs(chart_start(gpu, chart=2) - header.index("POWER") - width / 2) <= 1
+
+ def test_the_column_fits(self):
+ """`1000W/1000W` on a 2 TB host. Where a third chart doesn't fit, the column is left out
+ so that Rich doesn't cut the labels."""
+ job, metrics = make_run("ramp", gpus=2, memory_gb=2048)
+ metrics = with_power(metrics, 1000.0, limit=1000.0, gpus=2)
+ for width in range(57, 241, 5):
+ assert fits(render(job, metrics, width=width), width), width
+ assert row(render(job, metrics, width=120), "gpu=1").endswith("1000W/1000W")
+ assert "POWER" not in render(job, metrics, width=70)
+
+
@pytest.mark.parametrize(
"bytes_value,decimal_places,expected",
[
diff --git a/src/tests/_internal/server/background/scheduled_tasks/test_metrics.py b/src/tests/_internal/server/background/scheduled_tasks/test_metrics.py
index 59ac8a9c8c..65ecc7392e 100644
--- a/src/tests/_internal/server/background/scheduled_tasks/test_metrics.py
+++ b/src/tests/_internal/server/background/scheduled_tasks/test_metrics.py
@@ -1,4 +1,5 @@
from datetime import datetime, timezone
+from typing import Optional
from unittest.mock import patch
import pytest
@@ -35,7 +36,40 @@
class TestCollectMetrics:
@pytest.mark.asyncio
@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
- async def test_collects_metrics(self, test_db, session: AsyncSession):
+ @pytest.mark.parametrize(
+ ["gpus", "power_draw", "power_limit"],
+ [
+ pytest.param(
+ [
+ GPUMetrics(
+ gpu_memory_usage_bytes=0,
+ gpu_util_percent=0,
+ gpu_power_draw_watts=72.5,
+ gpu_power_limit_watts=300.0,
+ ),
+ # a GPU that prints [N/A] for power
+ GPUMetrics(gpu_memory_usage_bytes=0, gpu_util_percent=0),
+ ],
+ "[72.5, null]",
+ "[300.0, null]",
+ id="nvidia",
+ ),
+ pytest.param(
+ [GPUMetrics(gpu_memory_usage_bytes=0, gpu_util_percent=0)],
+ None,
+ None,
+ id="no-power-readings",
+ ),
+ ],
+ )
+ async def test_collects_metrics(
+ self,
+ test_db,
+ session: AsyncSession,
+ gpus: list[GPUMetrics],
+ power_draw: Optional[str],
+ power_limit: Optional[str],
+ ):
user = await create_user(session=session, global_role=GlobalRole.USER)
project = await create_project(session=session, owner=user)
await add_project_member(
@@ -76,12 +110,7 @@ async def test_collects_metrics(self, test_db, session: AsyncSession):
cpu_usage_micro=2,
memory_usage_bytes=3,
memory_working_set_bytes=4,
- gpus=[
- GPUMetrics(
- gpu_memory_usage_bytes=0,
- gpu_util_percent=0,
- )
- ],
+ gpus=gpus,
)
await collect_metrics()
SSHTunnelMock.assert_called_once()
@@ -89,6 +118,8 @@ async def test_collects_metrics(self, test_db, session: AsyncSession):
res = await session.execute(select(JobMetricsPoint))
metrics_point = res.scalar_one()
assert metrics_point.job_id == job.id
+ assert metrics_point.gpus_power_draw_watts == power_draw
+ assert metrics_point.gpus_power_limit_watts == power_limit
@pytest.mark.asyncio
@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
diff --git a/src/tests/_internal/server/routers/test_prometheus.py b/src/tests/_internal/server/routers/test_prometheus.py
index 26da89a90e..9a1131c6b2 100644
--- a/src/tests/_internal/server/routers/test_prometheus.py
+++ b/src/tests/_internal/server/routers/test_prometheus.py
@@ -152,6 +152,7 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient)
memory_usage_bytes=4_294_967_296,
gpus_util_percent=[80, 90],
gpus_memory_usage_bytes=[1_073_741_824, 2_147_483_648],
+ gpus_power_draw_watts=[52.5, None],
)
# Older, ignored
await create_job_metrics_point(
@@ -304,6 +305,9 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient)
# TYPE dstack_job_gpu_memory_usage_bytes gauge
dstack_job_gpu_memory_usage_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4",dstack_gpu_num="0"}} 1073741824.0
dstack_job_gpu_memory_usage_bytes{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4",dstack_gpu_num="1"}} 2147483648.0
+ # HELP dstack_job_gpu_power_draw_watts Power draw of the job's GPU, watts (NVIDIA only)
+ # TYPE dstack_job_gpu_power_draw_watts gauge
+ dstack_job_gpu_power_draw_watts{{dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4",dstack_gpu_num="0"}} 52.5
# HELP FIELD_1 Test field 1
# TYPE FIELD_1 gauge
FIELD_1{{gpu="0",dstack_project_name="project-1",dstack_user_name="test-user",dstack_run_name="run-1",dstack_run_id="{job_1_1.run_id}",dstack_job_name="run-1-0-0",dstack_job_id="{job_1_1.id}",dstack_job_num="0",dstack_replica_num="0",dstack_run_type="dev-environment",dstack_backend="aws",dstack_gpu="V4"}} 350.0
@@ -323,6 +327,33 @@ async def test_returns_metrics(self, session: AsyncSession, client: AsyncClient)
)
assert response.text.strip() == expected
+ async def test_skips_gpu_power_if_point_has_none(
+ self, session: AsyncSession, client: AsyncClient
+ ):
+ """Points written before the power columns were added, or by an older server replica,
+ have no power, and the other GPU metrics are still exported."""
+ user = await create_user(session=session, name="test-user", global_role=GlobalRole.USER)
+ project = await _create_project(session, "project-1", user)
+ job = await _create_job(
+ session=session,
+ run_name="run-1",
+ project=project,
+ user=user,
+ status=JobStatus.RUNNING,
+ job_provisioning_data=get_job_provisioning_data(gpu_count=2),
+ )
+ await create_job_metrics_point(
+ session=session,
+ job_model=job,
+ timestamp=FAKE_NOW - timedelta(seconds=30),
+ gpus_util_percent=[80, 90],
+ gpus_memory_usage_bytes=[1024, 2048],
+ )
+ response = await client.get("/metrics")
+ assert response.status_code == 200
+ assert response.text.count("dstack_job_gpu_usage_ratio{") == 2
+ assert "dstack_job_gpu_power_draw_watts{" not in response.text
+
@patch("prometheus_client.generate_latest", lambda: BASE_CLIENT_METRICS)
async def test_returns_empty_response_if_no_runs(self, client: AsyncClient):
response = await client.get("/metrics")
diff --git a/src/tests/_internal/server/services/test_metrics.py b/src/tests/_internal/server/services/test_metrics.py
index 083855652a..a20fc6ad30 100644
--- a/src/tests/_internal/server/services/test_metrics.py
+++ b/src/tests/_internal/server/services/test_metrics.py
@@ -165,3 +165,47 @@ async def test_get_metrics(
Metric(name="gpu_util_percent_gpu0", timestamps=ts, values=gpu0_util),
Metric(name="gpu_util_percent_gpu1", timestamps=ts, values=gpu1_util),
]
+
+ async def test_get_metrics_gpu_power(self, session: AsyncSession):
+ user = await create_user(session=session)
+ project = await create_project(session=session, owner=user)
+ repo = await create_repo(session=session, project_id=project.id)
+ run = await create_run(session=session, project=project, repo=repo, user=user)
+ jpd = get_job_provisioning_data(gpu_count=2)
+ job = await create_job(session=session, run=run, job_provisioning_data=jpd)
+ # gpu1 never reports power, gpu0 misses one reading, and the two oldest points have no
+ # power columns (the oldest one is only used for the CPU usage of the next one)
+ power = [
+ ([300.5, None], [400.0, None]),
+ ([None, None], [None, None]),
+ ([250.0, None], [400.0, None]),
+ (None, None),
+ (None, None),
+ ]
+ for dt, (draw, limit) in zip(self.ts, power):
+ await create_job_metrics_point(
+ session=session,
+ job_model=job,
+ timestamp=dt,
+ gpus_memory_usage_bytes=[1024, 1024],
+ gpus_util_percent=[50, 0],
+ gpus_power_draw_watts=draw,
+ gpus_power_limit_watts=limit,
+ )
+
+ metrics = await get_job_metrics(session, job)
+
+ assert [m for m in metrics.metrics if m.name.startswith("gpu_power")] == [
+ Metric(
+ name="gpu_power_draw_watts_gpu0",
+ timestamps=[self.ts[0], self.ts[2]],
+ values=[300.5, 250.0],
+ ),
+ Metric(
+ name="gpu_power_limit_watts_gpu0",
+ timestamps=[self.ts[0], self.ts[2]],
+ values=[400.0, 400.0],
+ ),
+ ]
+ util = next(m for m in metrics.metrics if m.name == "gpu_util_percent_gpu0")
+ assert len(util.timestamps) == 4