racer: fill the gaps in GPUDirect model loading

.github/workflows/ci.yaml

@@ -369,12 +369,16 @@ jobs:
           - configuration: simulation
             features: simulation
             native: false
+          # native compiles the decrypt kernels with nvcc; native-and-simulation
+          # keeps covering the build without them.
           - configuration: native
             features: native
             native: true
+            kernels: true
           - configuration: native-and-simulation
             features: native,simulation
             native: true
+            kernels: false
     env:
       CARGO_TARGET_DIR: ${{ github.workspace }}/bin/racer-gpu-cargo
       CARGO_BUILD_JOBS: "2"
@@ -387,11 +391,26 @@ jobs:
       - name: Set up pinned Rust
         run: timeout --signal=TERM --kill-after=10s 300s rustup toolchain install 1.96.0 --profile minimal --component rustfmt,clippy
 
-      # No CUDA toolkit on hosted runners: the shim builds without decrypt
-      # kernels and loads without a driver. Kernel math is covered by the
-      # C-core tests below and the kernel emulation step.
+      # Hosted runners have no GPU. nvcc (same package as the dataplane image)
+      # compiles the decrypt kernels to PTX so they at least build; running
+      # them needs .github/workflows/racer-gpu.yaml on a GPU runner. Kernel
+      # math is covered by the C-core tests below and the kernel emulation step.
+      - name: Install nvcc
+        if: matrix.kernels
+        run: |
+          timeout --signal=TERM --kill-after=10s 120s curl -fsSL -o "$RUNNER_TEMP/cuda-keyring.deb" \
+            https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
+          sudo dpkg -i "$RUNNER_TEMP/cuda-keyring.deb"
+          timeout --signal=TERM --kill-after=10s 300s sudo apt-get update
+          timeout --signal=TERM --kill-after=10s 600s sudo apt-get install -y --no-install-recommends cuda-nvcc-12-9
+          echo /usr/local/cuda/bin >> "$GITHUB_PATH"
+
+      - name: Compile CUDA shim with kernels (not hardware validation)
+        if: matrix.kernels
+        run: RACER_GPU_REQUIRE_PTX=1 timeout --signal=TERM --kill-after=10s 300s sh images/racer-dataplane/build-native-cuda.sh cmd/racer-dataplane/gpu/native bin/racer-gpu-native/libracer_gpu_cuda.so.1
+
       - name: Compile CUDA shim without kernels (not hardware validation)
-        if: matrix.native
+        if: matrix.native && !matrix.kernels
         run: timeout --signal=TERM --kill-after=10s 300s sh images/racer-dataplane/build-native-cuda.sh cmd/racer-dataplane/gpu/native bin/racer-gpu-native/libracer_gpu_cuda.so.1
 
       - name: Require CUDA shim ABI and symbols (no hardware)

.github/workflows/nightly.yaml

@@ -268,6 +268,7 @@ jobs:
   #   - orca                          : deployed separately by deploy-orca
   #   - racer-object                  : injected sidecars and per-ClusterCache
   #                                     origin DaemonSets from the racer component
+  #   - racer-vllm                    : vLLM with the Racer loader; not deployed
   #
   # machine-ops-controller is omitted on purpose: the operator synthesizes
   # metalman per Site instead, so machine-ops is never rolled out here.
@@ -297,6 +298,9 @@ jobs:
             file: images/racer-dataplane/Containerfile
           - name: racer-object
             file: images/racer-object/Containerfile
+          - name: racer-vllm
+            file: images/racer-vllm/Containerfile
+            large: true
           - name: orca
             file: images/orca/Containerfile
           - name: inventory-aggregator
@@ -316,6 +320,14 @@ jobs:
       - name: Normalize container registry to lowercase
         run: echo "REGISTRY=${REGISTRY,,}" >> "$GITHUB_ENV"
 
+      - name: Free disk space for large images
+        # The vLLM base image alone is larger than a hosted runner's free disk.
+        if: matrix.component.large
+        run: |
+          set -euo pipefail
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          df -h /
+
       - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
 
       - name: Log in to ghcr.io

.github/workflows/racer-gpu.yaml

@@ -0,0 +1,73 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+name: racer gpu
+
+# Runs `make racer-gpu-test` on a GPU runner: the CUDA shim with its decrypt
+# kernels, device discovery, client allocation import, GPU decrypt against the
+# CPU result, and cuFile reads. Hosted runners have no GPU, so this runs only
+# when the repository variable RACER_GPU_RUNNER holds the runner labels as a
+# JSON array, for example ["self-hosted", "linux", "gpu"]. Without it the job
+# is skipped instead of queuing forever (forks, repos without GPU runners).
+#
+# The runner needs an NVIDIA driver, gcc, curl and sudo apt access (for nvcc
+# if it is not installed). Set RACER_GPU_CUFILE=require to fail when cuFile
+# cannot open, on runners with GPUDirect Storage.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "17 6 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: racer-gpu-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  gpu:
+    name: Racer GPU hardware tests
+    if: vars.RACER_GPU_RUNNER != ''
+    runs-on: ${{ fromJSON(vars.RACER_GPU_RUNNER || '["ubuntu-24.04"]') }}
+    timeout-minutes: 45
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/bin/racer-gpu-cargo
+      RACER_GPU_TESTS_CUFILE: ${{ vars.RACER_GPU_CUFILE }}
+      RACER_GPU_TEST_DIR: ${{ github.workspace }}/tmp/racer-gpu
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Show GPUs
+        run: timeout --signal=TERM --kill-after=10s 60s nvidia-smi
+
+      - name: Set up pinned Rust
+        run: |
+          if ! command -v rustup >/dev/null; then
+            timeout --signal=TERM --kill-after=10s 300s sh -c 'curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none'
+            echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+            export PATH="$HOME/.cargo/bin:$PATH"
+          fi
+          timeout --signal=TERM --kill-after=10s 300s rustup toolchain install 1.96.0 --profile minimal
+          rustup override set 1.96.0
+
+      - name: Install nvcc if missing
+        run: |
+          if command -v nvcc >/dev/null || [ -x /usr/local/cuda/bin/nvcc ]; then
+            echo /usr/local/cuda/bin >> "$GITHUB_PATH"
+            exit 0
+          fi
+          . /etc/os-release
+          repo="${ID}${VERSION_ID//./}/$(uname -m | sed 's/aarch64/sbsa/')"
+          timeout --signal=TERM --kill-after=10s 120s curl -fsSL -o "$RUNNER_TEMP/cuda-keyring.deb" \
+            "https://developer.download.nvidia.com/compute/cuda/repos/$repo/cuda-keyring_1.1-1_all.deb"
+          sudo dpkg -i "$RUNNER_TEMP/cuda-keyring.deb"
+          timeout --signal=TERM --kill-after=10s 300s sudo apt-get update
+          timeout --signal=TERM --kill-after=10s 600s sudo apt-get install -y --no-install-recommends cuda-nvcc-12-9
+          echo /usr/local/cuda/bin >> "$GITHUB_PATH"
+
+      - name: Run GPU hardware tests
+        run: |
+          mkdir -p "$RACER_GPU_TEST_DIR"
+          timeout --signal=TERM --kill-after=30s 2400s make racer-gpu-test RACER_GPU_TESTS=require

.github/workflows/release.yaml

@@ -26,12 +26,14 @@ name: unbounded release
 #         -> metalman-image (stamped with the matching default netboot image)
 #     -> component-images  (matrix: machina, machine-ops-controller, unbounded-operator,
 #                           token-refresher, orca, gantry, racer-dataplane,
-#                           racer-object, host-ubuntu2404,
+#                           racer-object, racer-vllm,
+#                           host-ubuntu2404,
 #                           host-azurelinux3, host-azurelinux3-nvidia,
 #                           inventory-aggregator, inventory-inspector,
 #                           inventory-viewer)
 #         -> gantry-chart  (OCI chart + signed downloadable archive)
 #     -> manifests-tarball (make release-manifests + cosign sign)
+#     -> python-packages   (racer-client and racer-vllm sdists and wheels)
 #     -> finalize          (attach extras + krew manifest)
 #
 # All artifacts land on a single GitHub release that is left as a draft.
@@ -867,6 +869,8 @@ jobs:
             file: images/gantry/Containerfile
             platforms: linux/amd64,linux/arm64
             trivy: true
+          # Includes GPU reads (CUDA shim and cuFile). The dataplane touches no
+          # GPU until a client sends the first gpu-read.
           - name: racer-dataplane
             file: images/racer-dataplane/Containerfile
             platforms: linux/amd64,linux/arm64
@@ -875,6 +879,13 @@ jobs:
             file: images/racer-object/Containerfile
             platforms: linux/amd64,linux/arm64
             trivy: true
+          # vLLM with racer-client and racer-vllm. The upstream vLLM base is
+          # not ours to patch, so it is not gated on Trivy.
+          - name: racer-vllm
+            file: images/racer-vllm/Containerfile
+            platforms: linux/amd64
+            trivy: false
+            large: true
           - name: host-ubuntu2404
             file: images/host-ubuntu2404/Containerfile
             platforms: linux/amd64,linux/arm64
@@ -905,6 +916,14 @@ jobs:
       - name: Normalize container registry to lowercase
         run: echo "REGISTRY=${REGISTRY,,}" >> "$GITHUB_ENV"
 
+      - name: Free disk space for large images
+        # The vLLM base image alone is larger than a hosted runner's free disk.
+        if: matrix.component.large
+        run: |
+          set -euo pipefail
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          df -h /
+
       - name: Install cosign
         uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
 
@@ -1143,6 +1162,38 @@ jobs:
           # Past the window the only recovery is a full re-run.
           retention-days: 7
 
+  # ---------------------------------------------------------------------------
+  # Build sdists and wheels for the Racer Python packages (racer-client and
+  # racer-vllm) and attach them to the draft release. The release is the only
+  # place they are published; nothing is uploaded to PyPI.
+  # ---------------------------------------------------------------------------
+  python-packages:
+    needs: version
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Build sdists and wheels
+        run: |
+          set -euo pipefail
+          python3 -m venv "$RUNNER_TEMP/build-venv"
+          "$RUNNER_TEMP/build-venv/bin/pip" install --quiet build==1.3.0
+          for package in racer racer-vllm; do
+            timeout --signal=TERM --kill-after=10s 300s \
+              "$RUNNER_TEMP/build-venv/bin/python" -m build --sdist --wheel \
+              --outdir dist/python "clients/python/$package"
+          done
+          ls -l dist/python
+
+      - name: Attach Python packages to the release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          TAG: ${{ needs.version.outputs.tag }}
+        run: |
+          set -euo pipefail
+          gh release upload "$TAG" dist/python/* --clobber
+
   # ---------------------------------------------------------------------------
   # Attach the manifests tarball + krew manifest to the draft release that
   # GoReleaser created and set release notes. The release stays a draft here;
@@ -1160,6 +1211,7 @@ jobs:
       - component-images
       - gantry-chart
       - manifests-tarball
+      - python-packages
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

Makefile

@@ -366,6 +366,7 @@ help: ## Show this help
 	@echo ""
 	@echo "Racer:"
 	@echo "  racer-test                       Test the Rust Racer workspace and Go Racer API and wire contracts"
+	@echo "  racer-gpu-test                   Build the CUDA shim with kernels and run GPU hardware tests"
 	@echo "  racer-envtest                    Run Racer API-server tests with KUBEBUILDER_ASSETS"
 	@echo "  racer-envtest-ci                 Provision pinned assets and run Racer API-server tests"
 	@echo "  racer-generate                   Generate Racer deepcopy and CRD artifacts"
@@ -575,6 +576,23 @@ racer-rust-test: ## Test the Rust Racer workspace, including the embedded cluste
 
 racer-test: racer-go-test racer-rust-test ## Check Racer
 
+RACER_GPU_NATIVE ?= $(CURDIR)/bin/racer-gpu-native
+# 1 skips when no GPU is found; require fails instead (dedicated GPU runners).
+RACER_GPU_TESTS ?= require
+
+.PHONY: racer-gpu-test
+racer-gpu-test: ## Build the CUDA shim with kernels (needs nvcc) and run GPU hardware tests (needs a GPU)
+	RACER_GPU_REQUIRE_PTX=1 timeout --signal=TERM --kill-after=10s 600s sh images/racer-dataplane/build-native-cuda.sh cmd/racer-dataplane/gpu/native "$(RACER_GPU_NATIVE)/libracer_gpu_cuda.so.1"
+	LD_LIBRARY_PATH="$(RACER_GPU_NATIVE)$${LD_LIBRARY_PATH:+:$$LD_LIBRARY_PATH}" timeout --signal=TERM --kill-after=10s 300s cargo test --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --no-default-features --features native --lib ffi::tests::required_native_shim_loads_abi_and_symbols -- --exact --ignored
+	LD_LIBRARY_PATH="$(RACER_GPU_NATIVE)$${LD_LIBRARY_PATH:+:$$LD_LIBRARY_PATH}" RACER_GPU_TESTS=$(RACER_GPU_TESTS) timeout --signal=TERM --kill-after=10s 900s cargo test --locked --manifest-path cmd/racer-dataplane/Cargo.toml -p racer-gpu --no-default-features --features native --test hardware -- --nocapture --test-threads=1
+	@if [ -n "$$RACER_GPU_E2E_SOCKET" ]; then \
+		test -x "$(RACER_PY_VENV)/bin/pytest" || { timeout --signal=TERM --kill-after=10s 120s python3 -m venv "$(RACER_PY_VENV)" && \
+			timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pip" install -q pytest; }; \
+		cd clients/python/racer && PYTHONPATH=src timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pytest" -q -p no:cacheprovider -m gpu -rs; \
+	elif [ "$$RACER_GPU_E2E" = require ]; then \
+		echo "RACER_GPU_E2E=require but RACER_GPU_E2E_SOCKET is unset" >&2; exit 1; \
+	else echo "RACER_GPU_E2E_SOCKET unset: skipping the Python gpu-read test against a live dataplane"; fi
+
 RACER_PY_VENV ?= $(CURDIR)/tmp/racer-python-venv
 
 .PHONY: racer-python-test
@@ -584,6 +602,14 @@ racer-python-test: ## Run the Racer Python client and vLLM plugin tests (no GPU
 	cd clients/python/racer && PYTHONPATH=src timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pytest" -q -p no:cacheprovider
 	cd clients/python/racer-vllm && PYTHONPATH=src:../racer/src timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pytest" -q -p no:cacheprovider
 
+.PHONY: racer-python-dist
+racer-python-dist: ## Build sdists and wheels for racer-client and racer-vllm into tmp/racer-python-dist
+	@test -x "$(RACER_PY_VENV)/bin/pyproject-build" || { timeout --signal=TERM --kill-after=10s 120s python3 -m venv "$(RACER_PY_VENV)" && \
+		timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/pip" install -q build; }
+	rm -rf tmp/racer-python-dist
+	timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/python" -m build -o tmp/racer-python-dist clients/python/racer
+	timeout --signal=TERM --kill-after=10s 300s "$(RACER_PY_VENV)/bin/python" -m build -o tmp/racer-python-dist clients/python/racer-vllm
+
 $(SETUP_ENVTEST):
 	@mkdir -p bin tmp/envtest-tools
 	TMPDIR="$(CURDIR)/tmp/envtest-tools" GOBIN="$(CURDIR)/bin" timeout --signal=TERM --kill-after=10s 300s $(GOCMD) install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(SETUP_ENVTEST_VERSION)