From 64a8a2b7f78eb5b93678ab3be3ed2f081faabbd2 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 10:26:58 -0700 Subject: [PATCH 1/6] CI: verify GPU driver installer SHA-256 before execution Signed-off-by: Ralf Juengling --- .github/workflows/test-wheel-linux.yml | 1 + .github/workflows/test-wheel-windows.yml | 1 + ci/test-matrix.yml | 12 +- ci/tools/install_gpu_driver.ps1 | 23 +++- ci/tools/install_gpu_driver.sh | 20 +-- .../tests/test_test_matrix_driver_hashes.py | 114 ++++++++++++++++++ 6 files changed, 153 insertions(+), 18 deletions(-) create mode 100644 ci/tools/tests/test_test_matrix_driver_hashes.py diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 66faca5b0ad..b3e3f94a720 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -150,6 +150,7 @@ jobs: if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: DRIVER: ${{ matrix.DRIVER }} + DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }} GPU_TYPE: ${{ matrix.GPU }} run: ./ci/tools/install_gpu_driver.sh diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index ffb58c95602..90d64adac98 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -123,6 +123,7 @@ jobs: if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: DRIVER: ${{ matrix.DRIVER }} + DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }} GPU_TYPE: ${{ matrix.GPU }} run: | ci/tools/install_gpu_driver.ps1 diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index f6cc5624786..3beaea94310 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -20,7 +20,9 @@ # - install that version via ci/tools/install_gpu_driver.sh (Linux) # or ci/tools/install_gpu_driver.ps1 (Windows) at the start of the # job. The matrix row is routed to the 'latest' runner image (the -# install scripts swap the driver themselves). +# install scripts swap the driver themselves). A version-string +# row must also set DRIVER_SHA256 (regenerate with: +# `curl -fsSL | sha256sum`). # - DRIVER: 'earliest' does not work with CUDA 12.9.1 # - DRIVER: a custom version is not supported with FLAVOR=wsl on Linux. # - ENV: optional map of extra environment variables to inject into the test @@ -50,10 +52,10 @@ linux: - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02', DRIVER_SHA256: '3034a054bb4cdf7752ff8dc272564cb105513804bff53538945901b16ca77463' } - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: '610.43.02' } + - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: '610.43.02', DRIVER_SHA256: '3034a054bb4cdf7752ff8dc272564cb105513804bff53538945901b16ca77463' } - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } @@ -101,7 +103,7 @@ linux: - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } } # nightly-numba-cuda - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '580.65.06', ENV: { MODE: 'nightly-numba-cuda' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '580.65.06', DRIVER_SHA256: '04b10867af585e765cfbfdcf39ed5f4bd112375bebab0172eaa187c6aa5024ff', ENV: { MODE: 'nightly-numba-cuda' } } - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } # nightly-numba-cuda-mlir (MLIR backend, linux-64 only) @@ -150,7 +152,7 @@ windows: - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } } # nightly-numba-cuda - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '596.36', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '596.36', DRIVER_SHA256: '99182c2fdf3f23c503c74608945d84be99f036b5ec969f2945062069a15acd2e', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } # nightly-numba-cuda-mlir (MLIR backend, win-64) - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } } - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } } diff --git a/ci/tools/install_gpu_driver.ps1 b/ci/tools/install_gpu_driver.ps1 index e61c6bbdbb1..db4ab4b6090 100644 --- a/ci/tools/install_gpu_driver.ps1 +++ b/ci/tools/install_gpu_driver.ps1 @@ -10,11 +10,13 @@ # right driver). # # Inputs (env): -# DRIVER Driver version, e.g. "610.47". Must NOT be 'latest' or -# 'earliest' -- those are runner-pre-installed and the -# workflow is expected to skip this script for them. -# GPU_TYPE Lower-case GPU label from the matrix (e.g. "l4", "rtx4090"). -# Selects the data-center vs desktop installer variant. +# DRIVER Driver version, e.g. "610.47". Must NOT be 'latest' or +# 'earliest' -- those are runner-pre-installed and the +# workflow is expected to skip this script for them. +# DRIVER_SHA256 Expected SHA-256 hex digest of the downloaded driver +# installer. +# GPU_TYPE Lower-case GPU label from the matrix (e.g. "l4", "rtx4090"). +# Selects the data-center vs desktop installer variant. # Install the driver function Install-Driver { @@ -68,6 +70,17 @@ function Install-Driver { $ProgressPreference = $ProgressPreference_tmp Write-Output 'Download complete!' + if (-not $env:DRIVER_SHA256) { + Write-Error "DRIVER_SHA256 env var is required to verify the driver installer." + exit 1 + } + $actual_sha256 = (Get-FileHash -Algorithm SHA256 -Path $filepath).Hash + if ($actual_sha256 -ne $env:DRIVER_SHA256) { + Write-Error "SHA256 mismatch for ${filepath}: expected $($env:DRIVER_SHA256), got $actual_sha256" + exit 1 + } + Write-Output "SHA256 verified: $actual_sha256" + # Install the file with the specified path from earlier Write-Output 'Running the driver installer...' Start-Process -FilePath $filepath -ArgumentList $install_args -Wait diff --git a/ci/tools/install_gpu_driver.sh b/ci/tools/install_gpu_driver.sh index 777edd63519..8435854c7d3 100755 --- a/ci/tools/install_gpu_driver.sh +++ b/ci/tools/install_gpu_driver.sh @@ -13,13 +13,15 @@ # bind-mount refresh) and may cause issues. # # Inputs (env): -# DRIVER Driver version, e.g. "580.65.06". Must NOT be 'latest' or -# 'earliest' -- those are runner-pre-installed and the -# workflow is expected to skip this script for them. -# GPU_TYPE Lower-case GPU label from the matrix (e.g. "v100", "l4", -# "h100"). Used only to pick the kernel module flavor -# (Volta needs the proprietary/legacy module; everything -# newer can use the open module). +# DRIVER Driver version, e.g. "580.65.06". Must NOT be 'latest' or +# 'earliest' -- those are runner-pre-installed and the +# workflow is expected to skip this script for them. +# DRIVER_SHA256 Expected lowercase SHA-256 hex digest of the downloaded +# driver installer. +# GPU_TYPE Lower-case GPU label from the matrix (e.g. "v100", "l4", +# "h100"). Used only to pick the kernel module flavor +# (Volta needs the proprietary/legacy module; everything +# newer can use the open module). # # Arch is detected from `uname -m`. # @@ -32,6 +34,7 @@ set -euo pipefail : "${DRIVER:?DRIVER env var is required (e.g. 580.65.06)}" +: "${DRIVER_SHA256:?DRIVER_SHA256 env var is required (64-char sha256 of the installer)}" : "${GPU_TYPE:?GPU_TYPE env var is required (e.g. l4)}" case "$DRIVER" in @@ -70,7 +73,7 @@ URL="https://us.download.nvidia.com/XFree86/${ARCH_DIR}/${VERSION}/NVIDIA-Linux- # Re-elevate to root if needed (sudo is preinstalled on the runner image). if [ "$(id -u)" != 0 ]; then - exec sudo -E DRIVER="$DRIVER" GPU_TYPE="$GPU_TYPE" "$0" "$@" + exec sudo -E DRIVER="$DRIVER" DRIVER_SHA256="$DRIVER_SHA256" GPU_TYPE="$GPU_TYPE" "$0" "$@" fi echo "install_gpu_driver.sh is ALPHA -- it performs live modifications to the host driver stack and may cause issues" >&2 @@ -111,6 +114,7 @@ host_install() { d=$(mktemp -d) ( cd "$d" \ && wget -q -O installer.run "$URL" \ + && echo "${DRIVER_SHA256} installer.run" | sha256sum --check --strict - \ && sh installer.run --silent --dkms --no-questions \ --accept-license --ui=none --no-cc-version-check --kernel-module-type="$KMT" ) modprobe nvidia nvidia_uvm nvidia_modeset diff --git a/ci/tools/tests/test_test_matrix_driver_hashes.py b/ci/tools/tests/test_test_matrix_driver_hashes.py new file mode 100644 index 00000000000..fa72041687a --- /dev/null +++ b/ci/tools/tests/test_test_matrix_driver_hashes.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Ensure custom DRIVER entries in ci/test-matrix.yml carry verified SHA-256 hashes.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] +MATRIX_PATH = REPO_ROOT / "ci" / "test-matrix.yml" +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +PREINSTALLED_DRIVERS = {"latest", "earliest"} +WINDOWS_DATA_CENTER_GPUS = {"a100", "h100", "l4", "t4", "v100", "rtxa6000", "rtx6000ada"} +WINDOWS_DESKTOP_GPUS = {"rtx2080", "rtx4090", "rtxpro6000"} + + +def _load_matrix() -> dict[str, object]: + return yaml.safe_load(MATRIX_PATH.read_text(encoding="utf-8")) + + +def _iter_rows(matrix: dict[str, object]): + for platform in ("linux", "windows"): + sections = matrix.get(platform, {}) + assert isinstance(sections, dict) + for section_name, rows in sections.items(): + assert isinstance(rows, list) + for index, row in enumerate(rows): + assert isinstance(row, dict) + yield platform, section_name, index, row + + +def _installer_identity(platform: str, row: dict[str, object]) -> tuple[str, str, str]: + arch = str(row["ARCH"]) + driver = str(row["DRIVER"]) + if platform == "linux": + return (platform, arch, driver) + gpu = str(row["GPU"]) + if gpu in WINDOWS_DATA_CENTER_GPUS: + variant = "data-center" + elif gpu in WINDOWS_DESKTOP_GPUS: + variant = "desktop" + else: + raise AssertionError(f"Unknown Windows GPU '{gpu}' in ci/test-matrix.yml") + return (f"{platform}-{variant}", arch, driver) + + +def validate_matrix_driver_hashes(matrix: dict[str, object]) -> None: + seen_hashes: dict[tuple[str, str, str], str] = {} + custom_rows = 0 + + for platform, section_name, index, row in _iter_rows(matrix): + driver = row.get("DRIVER") + assert isinstance(driver, str) and driver, f"{platform}.{section_name}[{index}] is missing a non-empty DRIVER" + if driver in PREINSTALLED_DRIVERS: + assert "DRIVER_SHA256" not in row, ( + f"{platform}.{section_name}[{index}] sets DRIVER={driver!r} and must not set DRIVER_SHA256" + ) + continue + + custom_rows += 1 + sha256 = row.get("DRIVER_SHA256") + assert isinstance(sha256, str) and SHA256_RE.match(sha256), ( + f"{platform}.{section_name}[{index}] (DRIVER={driver!r}) must set " + "DRIVER_SHA256 to a 64-character lowercase hex SHA-256 digest" + ) + + identity = _installer_identity(platform, row) + previous = seen_hashes.setdefault(identity, sha256) + assert sha256 == previous, f"Inconsistent DRIVER_SHA256 for {identity}: {sha256!r} != {previous!r}" + + assert custom_rows > 0, "Expected at least one custom DRIVER row in ci/test-matrix.yml" + + +@pytest.mark.agent_authored(model="gemini-4-argon") +def test_test_matrix_custom_drivers_have_valid_consistent_sha256() -> None: + validate_matrix_driver_hashes(_load_matrix()) + + +@pytest.mark.agent_authored(model="gemini-4-argon") +def test_missing_or_invalid_driver_sha256_is_rejected() -> None: + matrix = _load_matrix() + for _platform, _section_name, _index, row in _iter_rows(matrix): + if row["DRIVER"] not in PREINSTALLED_DRIVERS: + target_row = row + break + else: + pytest.fail("No custom DRIVER row found in ci/test-matrix.yml") + + original = target_row.pop("DRIVER_SHA256") + with pytest.raises(AssertionError, match="must set DRIVER_SHA256"): + validate_matrix_driver_hashes(matrix) + + target_row["DRIVER_SHA256"] = "UPPERCASE" + ("0" * 55) + with pytest.raises(AssertionError, match="64-character lowercase hex"): + validate_matrix_driver_hashes(matrix) + + target_row["DRIVER_SHA256"] = original + + +@pytest.mark.agent_authored(model="gemini-4-argon") +def test_conflicting_driver_sha256_for_same_installer_is_rejected() -> None: + matrix = _load_matrix() + linux_pr_rows = matrix["linux"]["pull-request"] + custom_rows = [row for row in linux_pr_rows if row["DRIVER"] == "610.43.02"] + assert len(custom_rows) >= 2 + + custom_rows[1]["DRIVER_SHA256"] = "0" * 64 + with pytest.raises(AssertionError, match="Inconsistent DRIVER_SHA256"): + validate_matrix_driver_hashes(matrix) From 08cba07dce2b67e3fca6421d5e26ab4a8900b5ac Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 14:10:05 -0700 Subject: [PATCH 2/6] CI: download and verify driver installer before removing host driver Signed-off-by: Ralf Juengling --- ci/tools/install_gpu_driver.ps1 | 9 +++++---- ci/tools/install_gpu_driver.sh | 13 +++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ci/tools/install_gpu_driver.ps1 b/ci/tools/install_gpu_driver.ps1 index db4ab4b6090..1957cddb541 100644 --- a/ci/tools/install_gpu_driver.ps1 +++ b/ci/tools/install_gpu_driver.ps1 @@ -28,6 +28,11 @@ function Install-Driver { exit 1 } + if (-not $env:DRIVER_SHA256) { + Write-Error "DRIVER_SHA256 env var is required to verify the driver installer." + exit 1 + } + # Get GPU type from environment variable $gpu_type = $env:GPU_TYPE @@ -70,10 +75,6 @@ function Install-Driver { $ProgressPreference = $ProgressPreference_tmp Write-Output 'Download complete!' - if (-not $env:DRIVER_SHA256) { - Write-Error "DRIVER_SHA256 env var is required to verify the driver installer." - exit 1 - } $actual_sha256 = (Get-FileHash -Algorithm SHA256 -Path $filepath).Hash if ($actual_sha256 -ne $env:DRIVER_SHA256) { Write-Error "SHA256 mismatch for ${filepath}: expected $($env:DRIVER_SHA256), got $actual_sha256" diff --git a/ci/tools/install_gpu_driver.sh b/ci/tools/install_gpu_driver.sh index 8435854c7d3..97f8850dd2a 100755 --- a/ci/tools/install_gpu_driver.sh +++ b/ci/tools/install_gpu_driver.sh @@ -92,6 +92,15 @@ in_container() { host_install() { apt-get -y install build-essential dkms "linux-headers-$(uname -r)" psmisc kmod + # Download and verify the installer before tearing down the existing + # host driver so a download failure or SHA-256 mismatch leaves the + # runner's current driver intact. + local d + d=$(mktemp -d) + ( cd "$d" \ + && wget -q -O installer.run "$URL" \ + && echo "${DRIVER_SHA256} installer.run" | sha256sum --check --strict - ) + systemctl stop nvidia-persistenced dcgm-exporter || true # if-test instead of `fuser ... || true` so a kill failure surfaces # (fuser exits 1 when nothing holds the device, which is the happy path). @@ -110,11 +119,7 @@ host_install() { | awk -v re="$KEEP_RE" '$0 !~ re' \ | xargs -r apt-get -y remove --purge || true - local d - d=$(mktemp -d) ( cd "$d" \ - && wget -q -O installer.run "$URL" \ - && echo "${DRIVER_SHA256} installer.run" | sha256sum --check --strict - \ && sh installer.run --silent --dkms --no-questions \ --accept-license --ui=none --no-cc-version-check --kernel-module-type="$KMT" ) modprobe nvidia nvidia_uvm nvidia_modeset From 2e775e11057fba1c6213aafff82edf78732b70d2 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 15:12:44 -0700 Subject: [PATCH 3/6] CI: simplify driver hash matrix test and clarify nsenter error message Signed-off-by: Ralf Juengling --- ci/tools/install_gpu_driver.sh | 5 +- .../tests/test_test_matrix_driver_hashes.py | 114 +++--------------- 2 files changed, 22 insertions(+), 97 deletions(-) diff --git a/ci/tools/install_gpu_driver.sh b/ci/tools/install_gpu_driver.sh index 97f8850dd2a..7edfaf03580 100755 --- a/ci/tools/install_gpu_driver.sh +++ b/ci/tools/install_gpu_driver.sh @@ -202,9 +202,10 @@ if [ -z "${_NVDRV_NSENTERED:-}" ] && in_container; then # ours lives in the GH workspace mount (container-only), so we pipe the # script body in via stdin instead -- the `< "$0"` fd is opened before # nsenter and stays valid across the namespace switch. Env vars (DRIVER, - # GPU_TYPE, _NVDRV_NSENTERED) are inherited by the host-side bash. + # DRIVER_SHA256, GPU_TYPE, _NVDRV_NSENTERED) are inherited by the + # host-side bash. _NVDRV_NSENTERED=1 nsenter -t 1 -m -p -n -i -u -- bash -s < "$0" \ - || { echo "::error::container needs 'options: --privileged --pid=host'" >&2; exit 1; } + || { echo "::error::host-side driver install failed (see output above; also check container has 'options: --privileged --pid=host')" >&2; exit 1; } refresh_container_libs # Re-bind /run/nvidia-persistenced from the host. The container's diff --git a/ci/tools/tests/test_test_matrix_driver_hashes.py b/ci/tools/tests/test_test_matrix_driver_hashes.py index fa72041687a..b6c2c226c2a 100644 --- a/ci/tools/tests/test_test_matrix_driver_hashes.py +++ b/ci/tools/tests/test_test_matrix_driver_hashes.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Ensure custom DRIVER entries in ci/test-matrix.yml carry verified SHA-256 hashes.""" +"""Ensure custom DRIVER entries in ci/test-matrix.yml carry valid, consistent SHA-256 hashes.""" from __future__ import annotations @@ -11,104 +11,28 @@ import pytest import yaml -REPO_ROOT = Path(__file__).resolve().parents[3] -MATRIX_PATH = REPO_ROOT / "ci" / "test-matrix.yml" +MATRIX_PATH = Path(__file__).resolve().parents[2] / "test-matrix.yml" SHA256_RE = re.compile(r"^[0-9a-f]{64}$") PREINSTALLED_DRIVERS = {"latest", "earliest"} -WINDOWS_DATA_CENTER_GPUS = {"a100", "h100", "l4", "t4", "v100", "rtxa6000", "rtx6000ada"} -WINDOWS_DESKTOP_GPUS = {"rtx2080", "rtx4090", "rtxpro6000"} - - -def _load_matrix() -> dict[str, object]: - return yaml.safe_load(MATRIX_PATH.read_text(encoding="utf-8")) - - -def _iter_rows(matrix: dict[str, object]): - for platform in ("linux", "windows"): - sections = matrix.get(platform, {}) - assert isinstance(sections, dict) - for section_name, rows in sections.items(): - assert isinstance(rows, list) - for index, row in enumerate(rows): - assert isinstance(row, dict) - yield platform, section_name, index, row - - -def _installer_identity(platform: str, row: dict[str, object]) -> tuple[str, str, str]: - arch = str(row["ARCH"]) - driver = str(row["DRIVER"]) - if platform == "linux": - return (platform, arch, driver) - gpu = str(row["GPU"]) - if gpu in WINDOWS_DATA_CENTER_GPUS: - variant = "data-center" - elif gpu in WINDOWS_DESKTOP_GPUS: - variant = "desktop" - else: - raise AssertionError(f"Unknown Windows GPU '{gpu}' in ci/test-matrix.yml") - return (f"{platform}-{variant}", arch, driver) - - -def validate_matrix_driver_hashes(matrix: dict[str, object]) -> None: - seen_hashes: dict[tuple[str, str, str], str] = {} - custom_rows = 0 - - for platform, section_name, index, row in _iter_rows(matrix): - driver = row.get("DRIVER") - assert isinstance(driver, str) and driver, f"{platform}.{section_name}[{index}] is missing a non-empty DRIVER" - if driver in PREINSTALLED_DRIVERS: - assert "DRIVER_SHA256" not in row, ( - f"{platform}.{section_name}[{index}] sets DRIVER={driver!r} and must not set DRIVER_SHA256" - ) - continue - - custom_rows += 1 - sha256 = row.get("DRIVER_SHA256") - assert isinstance(sha256, str) and SHA256_RE.match(sha256), ( - f"{platform}.{section_name}[{index}] (DRIVER={driver!r}) must set " - "DRIVER_SHA256 to a 64-character lowercase hex SHA-256 digest" - ) - - identity = _installer_identity(platform, row) - previous = seen_hashes.setdefault(identity, sha256) - assert sha256 == previous, f"Inconsistent DRIVER_SHA256 for {identity}: {sha256!r} != {previous!r}" - - assert custom_rows > 0, "Expected at least one custom DRIVER row in ci/test-matrix.yml" @pytest.mark.agent_authored(model="gemini-4-argon") def test_test_matrix_custom_drivers_have_valid_consistent_sha256() -> None: - validate_matrix_driver_hashes(_load_matrix()) - - -@pytest.mark.agent_authored(model="gemini-4-argon") -def test_missing_or_invalid_driver_sha256_is_rejected() -> None: - matrix = _load_matrix() - for _platform, _section_name, _index, row in _iter_rows(matrix): - if row["DRIVER"] not in PREINSTALLED_DRIVERS: - target_row = row - break - else: - pytest.fail("No custom DRIVER row found in ci/test-matrix.yml") - - original = target_row.pop("DRIVER_SHA256") - with pytest.raises(AssertionError, match="must set DRIVER_SHA256"): - validate_matrix_driver_hashes(matrix) - - target_row["DRIVER_SHA256"] = "UPPERCASE" + ("0" * 55) - with pytest.raises(AssertionError, match="64-character lowercase hex"): - validate_matrix_driver_hashes(matrix) - - target_row["DRIVER_SHA256"] = original - - -@pytest.mark.agent_authored(model="gemini-4-argon") -def test_conflicting_driver_sha256_for_same_installer_is_rejected() -> None: - matrix = _load_matrix() - linux_pr_rows = matrix["linux"]["pull-request"] - custom_rows = [row for row in linux_pr_rows if row["DRIVER"] == "610.43.02"] - assert len(custom_rows) >= 2 + matrix = yaml.safe_load(MATRIX_PATH.read_text(encoding="utf-8")) + seen_hashes: dict[tuple[str, str, str, str], str] = {} + custom_rows = 0 - custom_rows[1]["DRIVER_SHA256"] = "0" * 64 - with pytest.raises(AssertionError, match="Inconsistent DRIVER_SHA256"): - validate_matrix_driver_hashes(matrix) + for platform in ("linux", "windows"): + for section_name, rows in matrix[platform].items(): + for index, row in enumerate(rows): + driver = row["DRIVER"] + if driver in PREINSTALLED_DRIVERS: + assert "DRIVER_SHA256" not in row, f"{platform}.{section_name}[{index}] sets DRIVER_SHA256" + continue + custom_rows += 1 + sha256 = row.get("DRIVER_SHA256", "") + assert SHA256_RE.match(sha256), f"{platform}.{section_name}[{index}] has invalid DRIVER_SHA256" + key = (platform, str(row["ARCH"]), str(row["GPU"]), str(driver)) + assert seen_hashes.setdefault(key, sha256) == sha256, f"Conflicting DRIVER_SHA256 for {key}" + + assert custom_rows > 0 From 4d2aa832a759d8ff2d985107199dd7ccc3f6d4b9 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 15:23:24 -0700 Subject: [PATCH 4/6] CI: run GPU driver install scripts from the workflow revision Signed-off-by: Ralf Juengling --- .github/workflows/test-wheel-linux.yml | 10 +++++++++- .github/workflows/test-wheel-windows.yml | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index b3e3f94a720..543b7f943f5 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -146,13 +146,21 @@ jobs: dependencies: "jq wget libgl1 libegl1 g++ util-linux yq" dependent_exes: "jq wget yq" + - name: Checkout driver install scripts (workflow revision) + if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: .driver-tools + sparse-checkout: ci/tools + - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: DRIVER: ${{ matrix.DRIVER }} DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }} GPU_TYPE: ${{ matrix.GPU }} - run: ./ci/tools/install_gpu_driver.sh + run: ./.driver-tools/ci/tools/install_gpu_driver.sh - name: Ensure GPU is working run: nvidia-smi diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 90d64adac98..b72c0d96fd9 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -119,6 +119,14 @@ jobs: with: enable-apt: true + - name: Checkout driver install scripts (workflow revision) + if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: .driver-tools + sparse-checkout: ci/tools + - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: @@ -126,7 +134,7 @@ jobs: DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }} GPU_TYPE: ${{ matrix.GPU }} run: | - ci/tools/install_gpu_driver.ps1 + .driver-tools/ci/tools/install_gpu_driver.ps1 - name: Configure driver mode env: From 5151c9aa4b5c177c05f64d1d5a272dfbcd0e40f0 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 15:31:47 -0700 Subject: [PATCH 5/6] CI: document DRIVER_SHA256 ownership and update procedure in test-matrix.yml Signed-off-by: Ralf Juengling --- ci/test-matrix.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index 3beaea94310..36241d8c015 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -21,8 +21,15 @@ # or ci/tools/install_gpu_driver.ps1 (Windows) at the start of the # job. The matrix row is routed to the 'latest' runner image (the # install scripts swap the driver themselves). A version-string -# row must also set DRIVER_SHA256 (regenerate with: -# `curl -fsSL | sha256sum`). +# row must also set DRIVER_SHA256: the SHA-256 of the exact +# installer the script downloads (see the URL construction in +# install_gpu_driver.sh / .ps1; Windows data-center and desktop +# installers differ). Whoever changes DRIVER owns this value: +# compute it on a machine with an up-to-date trust store, e.g. +# `curl -fsSL | sha256sum`, and state in the PR +# how it was obtained; reviewers should recompute it. Treat a +# CI hash mismatch as possible tampering -- never copy the +# "got" value from a job log into this file. # - DRIVER: 'earliest' does not work with CUDA 12.9.1 # - DRIVER: a custom version is not supported with FLAVOR=wsl on Linux. # - ENV: optional map of extra environment variables to inject into the test From 06707f203d1c8e7890f0d8f922bfb7cc9e825c95 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 15:36:23 -0700 Subject: [PATCH 6/6] CI: tighten driver hash test key/regex and disable auxiliary checkout creds Signed-off-by: Ralf Juengling --- .github/workflows/test-wheel-linux.yml | 1 + .github/workflows/test-wheel-windows.yml | 1 + ci/tools/tests/test_test_matrix_driver_hashes.py | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 543b7f943f5..0582901a975 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -153,6 +153,7 @@ jobs: ref: ${{ github.sha }} path: .driver-tools sparse-checkout: ci/tools + persist-credentials: false - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index b72c0d96fd9..375dae14ef4 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -126,6 +126,7 @@ jobs: ref: ${{ github.sha }} path: .driver-tools sparse-checkout: ci/tools + persist-credentials: false - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} diff --git a/ci/tools/tests/test_test_matrix_driver_hashes.py b/ci/tools/tests/test_test_matrix_driver_hashes.py index b6c2c226c2a..23432964b2f 100644 --- a/ci/tools/tests/test_test_matrix_driver_hashes.py +++ b/ci/tools/tests/test_test_matrix_driver_hashes.py @@ -31,8 +31,8 @@ def test_test_matrix_custom_drivers_have_valid_consistent_sha256() -> None: continue custom_rows += 1 sha256 = row.get("DRIVER_SHA256", "") - assert SHA256_RE.match(sha256), f"{platform}.{section_name}[{index}] has invalid DRIVER_SHA256" - key = (platform, str(row["ARCH"]), str(row["GPU"]), str(driver)) + assert SHA256_RE.fullmatch(sha256), f"{platform}.{section_name}[{index}] has invalid DRIVER_SHA256" + key = (platform, str(row["ARCH"]), "" if platform == "linux" else str(row["GPU"]), str(driver)) assert seen_hashes.setdefault(key, sha256) == sha256, f"Conflicting DRIVER_SHA256 for {key}" assert custom_rows > 0