diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 66faca5b0ad..0582901a975 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -146,12 +146,22 @@ jobs: dependencies: "jq wget libgl1 libegl1 g++ util-linux yq" dependent_exes: "jq wget yq" + - name: Checkout driver install scripts (workflow revision) + if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: .driver-tools + sparse-checkout: ci/tools + persist-credentials: false + - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: DRIVER: ${{ matrix.DRIVER }} + DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }} GPU_TYPE: ${{ matrix.GPU }} - run: ./ci/tools/install_gpu_driver.sh + run: ./.driver-tools/ci/tools/install_gpu_driver.sh - name: Ensure GPU is working run: nvidia-smi diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index ffb58c95602..375dae14ef4 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -119,13 +119,23 @@ jobs: with: enable-apt: true + - name: Checkout driver install scripts (workflow revision) + if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: .driver-tools + sparse-checkout: ci/tools + persist-credentials: false + - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} env: DRIVER: ${{ matrix.DRIVER }} + DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }} GPU_TYPE: ${{ matrix.GPU }} run: | - ci/tools/install_gpu_driver.ps1 + .driver-tools/ci/tools/install_gpu_driver.ps1 - name: Configure driver mode env: diff --git a/ci/test-matrix.yml b/ci/test-matrix.yml index f6cc5624786..36241d8c015 100644 --- a/ci/test-matrix.yml +++ b/ci/test-matrix.yml @@ -20,7 +20,16 @@ # - install that version via ci/tools/install_gpu_driver.sh (Linux) # or ci/tools/install_gpu_driver.ps1 (Windows) at the start of the # job. The matrix row is routed to the 'latest' runner image (the -# install scripts swap the driver themselves). +# install scripts swap the driver themselves). A version-string +# row must also set DRIVER_SHA256: the SHA-256 of the exact +# installer the script downloads (see the URL construction in +# install_gpu_driver.sh / .ps1; Windows data-center and desktop +# installers differ). Whoever changes DRIVER owns this value: +# compute it on a machine with an up-to-date trust store, e.g. +# `curl -fsSL | sha256sum`, and state in the PR +# how it was obtained; reviewers should recompute it. Treat a +# CI hash mismatch as possible tampering -- never copy the +# "got" value from a job log into this file. # - DRIVER: 'earliest' does not work with CUDA 12.9.1 # - DRIVER: a custom version is not supported with FLAVOR=wsl on Linux. # - ENV: optional map of extra environment variables to inject into the test @@ -50,10 +59,10 @@ linux: - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'v100', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: '610.43.02', DRIVER_SHA256: '3034a054bb4cdf7752ff8dc272564cb105513804bff53538945901b16ca77463' } - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: '610.43.02' } + - { ARCH: 'amd64', PY_VER: '3.14', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: '610.43.02', DRIVER_SHA256: '3034a054bb4cdf7752ff8dc272564cb105513804bff53538945901b16ca77463' } - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '12.9.1', LOCAL_CTK: '1', GPU: 't4', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.0.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } - { ARCH: 'amd64', PY_VER: '3.14t', CUDA_VER: '13.4.2', LOCAL_CTK: '1', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest' } @@ -101,7 +110,7 @@ linux: - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } } # nightly-numba-cuda - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '580.65.06', ENV: { MODE: 'nightly-numba-cuda' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '580.65.06', DRIVER_SHA256: '04b10867af585e765cfbfdcf39ed5f4bd112375bebab0172eaa187c6aa5024ff', ENV: { MODE: 'nightly-numba-cuda' } } - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', ENV: { MODE: 'nightly-numba-cuda' } } # nightly-numba-cuda-mlir (MLIR backend, linux-64 only) @@ -150,7 +159,7 @@ windows: - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.0.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-pytorch', TORCH_VER: '2.9.1', TORCH_CUDA: 'cu130' } } # nightly-numba-cuda - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '596.36', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'l4', GPU_COUNT: '1', DRIVER: '596.36', DRIVER_SHA256: '99182c2fdf3f23c503c74608945d84be99f036b5ec969f2945062069a15acd2e', DRIVER_MODE: 'TCC', ENV: { MODE: 'nightly-numba-cuda' } } # nightly-numba-cuda-mlir (MLIR backend, win-64) - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } } - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '13.4.2', LOCAL_CTK: '0', GPU: 'rtxpro6000', GPU_COUNT: '1', DRIVER: 'latest', DRIVER_MODE: 'MCDM', ENV: { MODE: 'nightly-numba-cuda-mlir' } } diff --git a/ci/tools/install_gpu_driver.ps1 b/ci/tools/install_gpu_driver.ps1 index e61c6bbdbb1..1957cddb541 100644 --- a/ci/tools/install_gpu_driver.ps1 +++ b/ci/tools/install_gpu_driver.ps1 @@ -10,11 +10,13 @@ # right driver). # # Inputs (env): -# DRIVER Driver version, e.g. "610.47". Must NOT be 'latest' or -# 'earliest' -- those are runner-pre-installed and the -# workflow is expected to skip this script for them. -# GPU_TYPE Lower-case GPU label from the matrix (e.g. "l4", "rtx4090"). -# Selects the data-center vs desktop installer variant. +# DRIVER Driver version, e.g. "610.47". Must NOT be 'latest' or +# 'earliest' -- those are runner-pre-installed and the +# workflow is expected to skip this script for them. +# DRIVER_SHA256 Expected SHA-256 hex digest of the downloaded driver +# installer. +# GPU_TYPE Lower-case GPU label from the matrix (e.g. "l4", "rtx4090"). +# Selects the data-center vs desktop installer variant. # Install the driver function Install-Driver { @@ -26,6 +28,11 @@ function Install-Driver { exit 1 } + if (-not $env:DRIVER_SHA256) { + Write-Error "DRIVER_SHA256 env var is required to verify the driver installer." + exit 1 + } + # Get GPU type from environment variable $gpu_type = $env:GPU_TYPE @@ -68,6 +75,13 @@ function Install-Driver { $ProgressPreference = $ProgressPreference_tmp Write-Output 'Download complete!' + $actual_sha256 = (Get-FileHash -Algorithm SHA256 -Path $filepath).Hash + if ($actual_sha256 -ne $env:DRIVER_SHA256) { + Write-Error "SHA256 mismatch for ${filepath}: expected $($env:DRIVER_SHA256), got $actual_sha256" + exit 1 + } + Write-Output "SHA256 verified: $actual_sha256" + # Install the file with the specified path from earlier Write-Output 'Running the driver installer...' Start-Process -FilePath $filepath -ArgumentList $install_args -Wait diff --git a/ci/tools/install_gpu_driver.sh b/ci/tools/install_gpu_driver.sh index 777edd63519..7edfaf03580 100755 --- a/ci/tools/install_gpu_driver.sh +++ b/ci/tools/install_gpu_driver.sh @@ -13,13 +13,15 @@ # bind-mount refresh) and may cause issues. # # Inputs (env): -# DRIVER Driver version, e.g. "580.65.06". Must NOT be 'latest' or -# 'earliest' -- those are runner-pre-installed and the -# workflow is expected to skip this script for them. -# GPU_TYPE Lower-case GPU label from the matrix (e.g. "v100", "l4", -# "h100"). Used only to pick the kernel module flavor -# (Volta needs the proprietary/legacy module; everything -# newer can use the open module). +# DRIVER Driver version, e.g. "580.65.06". Must NOT be 'latest' or +# 'earliest' -- those are runner-pre-installed and the +# workflow is expected to skip this script for them. +# DRIVER_SHA256 Expected lowercase SHA-256 hex digest of the downloaded +# driver installer. +# GPU_TYPE Lower-case GPU label from the matrix (e.g. "v100", "l4", +# "h100"). Used only to pick the kernel module flavor +# (Volta needs the proprietary/legacy module; everything +# newer can use the open module). # # Arch is detected from `uname -m`. # @@ -32,6 +34,7 @@ set -euo pipefail : "${DRIVER:?DRIVER env var is required (e.g. 580.65.06)}" +: "${DRIVER_SHA256:?DRIVER_SHA256 env var is required (64-char sha256 of the installer)}" : "${GPU_TYPE:?GPU_TYPE env var is required (e.g. l4)}" case "$DRIVER" in @@ -70,7 +73,7 @@ URL="https://us.download.nvidia.com/XFree86/${ARCH_DIR}/${VERSION}/NVIDIA-Linux- # Re-elevate to root if needed (sudo is preinstalled on the runner image). if [ "$(id -u)" != 0 ]; then - exec sudo -E DRIVER="$DRIVER" GPU_TYPE="$GPU_TYPE" "$0" "$@" + exec sudo -E DRIVER="$DRIVER" DRIVER_SHA256="$DRIVER_SHA256" GPU_TYPE="$GPU_TYPE" "$0" "$@" fi echo "install_gpu_driver.sh is ALPHA -- it performs live modifications to the host driver stack and may cause issues" >&2 @@ -89,6 +92,15 @@ in_container() { host_install() { apt-get -y install build-essential dkms "linux-headers-$(uname -r)" psmisc kmod + # Download and verify the installer before tearing down the existing + # host driver so a download failure or SHA-256 mismatch leaves the + # runner's current driver intact. + local d + d=$(mktemp -d) + ( cd "$d" \ + && wget -q -O installer.run "$URL" \ + && echo "${DRIVER_SHA256} installer.run" | sha256sum --check --strict - ) + systemctl stop nvidia-persistenced dcgm-exporter || true # if-test instead of `fuser ... || true` so a kill failure surfaces # (fuser exits 1 when nothing holds the device, which is the happy path). @@ -107,10 +119,7 @@ host_install() { | awk -v re="$KEEP_RE" '$0 !~ re' \ | xargs -r apt-get -y remove --purge || true - local d - d=$(mktemp -d) ( cd "$d" \ - && wget -q -O installer.run "$URL" \ && sh installer.run --silent --dkms --no-questions \ --accept-license --ui=none --no-cc-version-check --kernel-module-type="$KMT" ) modprobe nvidia nvidia_uvm nvidia_modeset @@ -193,9 +202,10 @@ if [ -z "${_NVDRV_NSENTERED:-}" ] && in_container; then # ours lives in the GH workspace mount (container-only), so we pipe the # script body in via stdin instead -- the `< "$0"` fd is opened before # nsenter and stays valid across the namespace switch. Env vars (DRIVER, - # GPU_TYPE, _NVDRV_NSENTERED) are inherited by the host-side bash. + # DRIVER_SHA256, GPU_TYPE, _NVDRV_NSENTERED) are inherited by the + # host-side bash. _NVDRV_NSENTERED=1 nsenter -t 1 -m -p -n -i -u -- bash -s < "$0" \ - || { echo "::error::container needs 'options: --privileged --pid=host'" >&2; exit 1; } + || { echo "::error::host-side driver install failed (see output above; also check container has 'options: --privileged --pid=host')" >&2; exit 1; } refresh_container_libs # Re-bind /run/nvidia-persistenced from the host. The container's diff --git a/ci/tools/tests/test_test_matrix_driver_hashes.py b/ci/tools/tests/test_test_matrix_driver_hashes.py new file mode 100644 index 00000000000..23432964b2f --- /dev/null +++ b/ci/tools/tests/test_test_matrix_driver_hashes.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Ensure custom DRIVER entries in ci/test-matrix.yml carry valid, consistent SHA-256 hashes.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +MATRIX_PATH = Path(__file__).resolve().parents[2] / "test-matrix.yml" +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +PREINSTALLED_DRIVERS = {"latest", "earliest"} + + +@pytest.mark.agent_authored(model="gemini-4-argon") +def test_test_matrix_custom_drivers_have_valid_consistent_sha256() -> None: + matrix = yaml.safe_load(MATRIX_PATH.read_text(encoding="utf-8")) + seen_hashes: dict[tuple[str, str, str, str], str] = {} + custom_rows = 0 + + for platform in ("linux", "windows"): + for section_name, rows in matrix[platform].items(): + for index, row in enumerate(rows): + driver = row["DRIVER"] + if driver in PREINSTALLED_DRIVERS: + assert "DRIVER_SHA256" not in row, f"{platform}.{section_name}[{index}] sets DRIVER_SHA256" + continue + custom_rows += 1 + sha256 = row.get("DRIVER_SHA256", "") + assert SHA256_RE.fullmatch(sha256), f"{platform}.{section_name}[{index}] has invalid DRIVER_SHA256" + key = (platform, str(row["ARCH"]), "" if platform == "linux" else str(row["GPU"]), str(driver)) + assert seen_hashes.setdefault(key, sha256) == sha256, f"Conflicting DRIVER_SHA256 for {key}" + + assert custom_rows > 0