Add fabric manager configuration support #2045

cdesiniotis · 2026-01-28T17:43:43Z

Question (for my understanding) -- GPUs may not be bound to the nvidia driver since there is a chance that the vfio-manager ran already and unbound the devices? Am I understanding this correct?

-Original file line number
+Diff line change
@@ Expand Up / @@ -94,6 +94,8 @@ type ClusterPolicySpec struct { @@
     	CCManager CCManagerSpec `json:"ccManager,omitempty"`
     	// HostPaths defines various paths on the host needed by GPU Operator components
     	HostPaths HostPathsSpec `json:"hostPaths,omitempty"`
+    	// FabricManager component spec
+    	FabricManager FabricManagerSpec `json:"fabricManager,omitempty"`
     }
     // Runtime defines container runtime type
@@ Expand Down Expand Up / @@ -1724,6 +1726,38 @@ type CDIConfigSpec struct { @@
     	Default *bool `json:"default,omitempty"`
     }
+    // FabricMode defines the Fabric Manager mode
+    type FabricMode string
+    const (
+    	// FabricModeFullPassthrough indicates Full-passthrough mode (FABRIC_MODE=0)
+    	FabricModeFullPassthrough FabricMode = "full-passthrough"
+    	// FabricModeSharedNVSwitch indicates Shared NVSwitch Virtualization mode (FABRIC_MODE=1)
+    	FabricModeSharedNVSwitch FabricMode = "shared-nvswitch"
+    )
+    func (f FabricMode) String() string {
+    	switch f {
+    	case FabricModeFullPassthrough:
+    		return "full-passthrough"
+    	case FabricModeSharedNVSwitch:
+    		return "shared-nvswitch"
+    	default:
+    		return ""
+    	}
+    }
+    // FabricManagerSpec defines the properties for NVIDIA Fabric Manager configuration
+    type FabricManagerSpec struct {
+    	// Mode indicates the Fabric Manager mode
+    	// +kubebuilder:validation:Enum=full-passthrough;shared-nvswitch
+    	// +kubebuilder:default=full-passthrough
+    	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
+    	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Fabric Manager Mode"
+    	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:select:full-passthrough,urn:alm:descriptor:com.tectonic.ui:select:shared-nvswitch"
+    	Mode FabricMode `json:"mode,omitempty"`
+    }
     // MIGStrategy indicates MIG mode
     type MIGStrategy string
@@ Expand Down Expand Up / @@ -2218,3 +2252,18 @@ func (c *MIGPartedConfigSpec) GetName() string { @@
     func (c *VGPUDevicesConfigSpec) GetName() string {
     	return ptr.Deref(c, VGPUDevicesConfigSpec{}).Name
     }
+    // IsSharedNVSwitchMode returns true if Fabric Manager is configured for Shared NVSwitch mode
+    func (f *FabricManagerSpec) IsSharedNVSwitchMode() bool {
+    	return f.Mode == FabricModeSharedNVSwitch
+    }
+    // ValidateFabricManagerConfig validates the Fabric Manager configuration
+    func (c *ClusterPolicySpec) ValidateFabricManagerConfig() error {
+    	if c.SandboxWorkloads.DefaultWorkload == "vm-passthrough" &&
+    		c.FabricManager.IsSharedNVSwitchMode() &&
+    		!c.Driver.IsEnabled() {
+    		return fmt.Errorf("driver must be enabled when using vm-passthrough with Fabric Manager Shared NVSwitch mode")
+    	}
+    	return nil
+    }

-Original file line number
+Diff line change
@@ Expand Up / @@ -22,8 +22,14 @@ data: @@
         fi
         if ! nvidia-smi; then
-          echo "nvidia-smi failed"
-          exit 1
+          # For vm-passthrough with shared-nvswitch mode, nvidia-smi may fail due to unbound devices
+          # Fall back to checking if nvidia module is loaded when FABRIC_MANAGER_FABRIC_MODE=1
+          if [ "${FABRIC_MANAGER_FABRIC_MODE:-}" = "1" ]; then
+            echo "nvidia-smi failed but nvidia module is loaded (vm-passthrough with shared-nvswitch mode)"
+          else
+            echo "nvidia-smi failed"
+            exit 1
+          fi
         fi
         GPU_DIRECT_RDMA_ENABLED="${GPU_DIRECT_RDMA_ENABLED:-false}"
@@ Expand Down @@

-Original file line number
+Diff line change
@@ Expand Up / @@ -12,3 +12,10 @@ rules: @@
       - use
       resourceNames:
       - privileged
+    - apiGroups:
+      - apps
+      resources:
+      - daemonsets
+      verbs:
+      - get
+      - list

-Original file line number
+Diff line change
@@ Expand Up / @@ -26,6 +26,36 @@ spec: @@
           priorityClassName: system-node-critical
           serviceAccountName: nvidia-sandbox-validator
           initContainers:
+            - name: driver-validation
+              image: "FILLED BY THE OPERATOR"
+              command: ["sh", "-c"]
+              args: ["nvidia-validator"]
+              env:
+                - name: WITH_WAIT
+                  value: "true"
+                - name: COMPONENT
+                  value: driver
+                - name: OPERATOR_NAMESPACE
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.namespace
+              securityContext:
+                privileged: true
+                seLinuxOptions:
+                  level: "s0"
+              volumeMounts:
+                - name: host-root
+                  mountPath: /host
+                  readOnly: true
+                  mountPropagation: HostToContainer
+                - name: driver-install-path
+                  mountPath: /run/nvidia/driver
+                  mountPropagation: HostToContainer
+                - name: run-nvidia-validations
+                  mountPath: /run/nvidia/validations
+                  mountPropagation: Bidirectional
+                - name: host-dev-char
+                  mountPath: /host-dev-char
             - name: cc-manager-validation
               image: "FILLED BY THE OPERATOR"
               command: ['sh', '-c']
@@ Expand Down Expand Up / @@ -145,3 +175,6 @@ spec: @@
             - name: host-root
               hostPath:
                 path: /
+            - name: host-dev-char
+              hostPath:
+                path: /dev/char

-Original file line number
+Diff line change
@@ Expand Up / @@ -80,6 +80,9 @@ spec: @@
                 readOnly: true
               - name: host-root
                 mountPath: /host
+              - name: run-nvidia-validations
+                mountPath: /run/nvidia/validations
+                mountPropagation: Bidirectional
               securityContext:
                 privileged: true
                 seLinuxOptions:
@@ Expand All / @@ -102,6 +105,10 @@ spec: @@
               hostPath:
                 path: /run/nvidia
                 type: DirectoryOrCreate
+            - name: run-nvidia-validations
+              hostPath:
+                path: /run/nvidia/validations
+                type: DirectoryOrCreate
             - name: host-root
               hostPath:
                 path: "/"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add fabric manager configuration support #2045

Diff view

Diff view

There are no files selected for viewing

Uh oh!

cdesiniotis Jan 28, 2026

Uh oh!

Uh oh!

-Original file line number
+Diff line change
@@ Expand Up / @@ -1057,6 +1057,17 @@ spec: @@
                             type: string
                         type: object
                     type: object
+                  fabricManager:
+                    description: FabricManager component spec
+                    properties:
+                      mode:
+                        default: full-passthrough
+                        description: Mode indicates the Fabric Manager mode
+                        enum:
+                        - full-passthrough
+                        - shared-nvswitch
+                        type: string
+                    type: object
                   gdrcopy:
                     description: GDRCopy component spec
                     properties:
@@ Expand Down @@

-Original file line number
+Diff line change
@@ Expand Up / @@ -1624,18 +1624,22 @@ func (v *VfioPCI) validate() error { @@
     		return err
     	}
-    	err = v.runValidation()
-    	if err != nil {
-    		return err
-    	}
-    	log.Info("Validation completed successfully - all devices are bound to vfio-pci")
+    	for {
+    		log.Info("Attempting to validate that all device are bound to vfio-pci")
+    		err := v.runValidation()
+    		if err != nil {
+    			if !withWaitFlag {
+    				return fmt.Errorf("error validating vfio-pci: %w", err)
+    			}
+    			log.Warningf("failed to validate vfio-pci, retrying after %d seconds\n", sleepIntervalSecondsFlag)
+    			time.Sleep(time.Duration(sleepIntervalSecondsFlag) * time.Second)
+    			continue
+    		}
-    	// delete status file is already present
-    	err = createStatusFile(outputDirFlag + "/" + vfioPCIStatusFile)
-    	if err != nil {
-    		return err
+    		log.Info("Validation completed successfully - all devices are bound to vfio-pci")
+    		return createStatusFile(outputDirFlag + "/" + vfioPCIStatusFile)
     	}
-    	return nil
     }
     func (v *VfioPCI) runValidation() error {
@@ Expand Down @@

Add fabric manager configuration support #2045

Are you sure you want to change the base?

Add fabric manager configuration support #2045

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

cdesiniotis Jan 28, 2026

Choose a reason for hiding this comment

Uh oh!

Uh oh!