gpu: add support for the upcoming xe-driver

Plugin can support both i915 and xe drivers dynamically. But
having both drivers on same node with RM is not possible.

Signed-off-by: Tuomas Katila <tuomas.katila@intel.com>

Author: tkatila

.github/workflows/lib-e2e.yaml

@@ -25,6 +25,7 @@ jobs:
           - name: e2e-gpu
             runner: gpu
             images: intel-gpu-plugin intel-gpu-initcontainer
+            targetJob: e2e-gpu SKIP=Resource:xe
           - name: e2e-iaa-spr
             targetjob: e2e-iaa
             runner: simics-spr

cmd/gpu_plugin/device_props.go

@@ -0,0 +1,85 @@
+// Copyright 2024 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"slices"
+
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler"
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
+	"k8s.io/klog/v2"
+)
+
+type DeviceProperties struct {
+	currentDriver string
+	drmDrivers    map[string]bool
+	tileCounts    []uint64
+	isPfWithVfs   bool
+}
+
+type invalidTileCountErr struct {
+	error
+}
+
+func newDeviceProperties() *DeviceProperties {
+	return &DeviceProperties{
+		drmDrivers: make(map[string]bool),
+	}
+}
+
+func (d *DeviceProperties) fetch(cardPath string) {
+	d.isPfWithVfs = pluginutils.IsSriovPFwithVFs(cardPath)
+
+	d.tileCounts = append(d.tileCounts, labeler.GetTileCount(cardPath))
+
+	driverName, err := pluginutils.ReadDeviceDriver(cardPath)
+	if err != nil {
+		klog.Warningf("card (%s) doesn't have driver, using default: %s", cardPath, deviceTypeDefault)
+
+		driverName = deviceTypeDefault
+	}
+
+	d.currentDriver = driverName
+	d.drmDrivers[d.currentDriver] = true
+}
+
+func (d *DeviceProperties) drmDriverCount() int {
+	return len(d.drmDrivers)
+}
+
+func (d *DeviceProperties) driver() string {
+	return d.currentDriver
+}
+
+func (d *DeviceProperties) monitorResource() string {
+	return d.currentDriver + monitorSuffix
+}
+
+func (d *DeviceProperties) maxTileCount() (uint64, error) {
+	if len(d.tileCounts) == 0 {
+		return 0, invalidTileCountErr{}
+	}
+
+	minCount := slices.Min(d.tileCounts)
+	maxCount := slices.Max(d.tileCounts)
+
+	if minCount != maxCount {
+		klog.Warningf("Node's GPUs are heterogenous (min: %d, max: %d tiles)", minCount, maxCount)
+
+		return 0, invalidTileCountErr{}
+	}
+
+	return maxCount, nil
+}

cmd/gpu_plugin/gpu_plugin.go

@@ -17,6 +17,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -32,7 +33,6 @@ import (
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/rm"
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler"
-	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 )
 
@@ -47,12 +47,14 @@ const (
 	vendorString      = "0x8086"
 
 	// Device plugin settings.
-	namespace  = "gpu.intel.com"
-	deviceType = "i915"
+	namespace         = "gpu.intel.com"
+	deviceTypeI915    = "i915"
+	deviceTypeXe      = "xe"
+	deviceTypeDefault = deviceTypeI915
 
 	// telemetry resource settings.
-	monitorType = "i915_monitoring"
-	monitorID   = "all"
+	monitorSuffix = "_monitoring"
+	monitorID     = "all"
 
 	// Period of device scans.
 	scanPeriod = 5 * time.Second
@@ -68,6 +70,10 @@ type cliOptions struct {
 	resourceManagement        bool
 }
 
+type rmWithMultipleDriversErr struct {
+	error
+}
+
 type preferredAllocationPolicyFunc func(*pluginapi.ContainerPreferredAllocationRequest) []string
 
 // nonePolicy is used for allocating GPU devices randomly, while trying
@@ -283,7 +289,11 @@ func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugi
 	if options.resourceManagement {
 		var err error
 
-		dp.resMan, err = rm.NewResourceManager(monitorID, namespace+"/"+deviceType)
+		dp.resMan, err = rm.NewResourceManager(monitorID,
+			[]string{
+				namespace + "/" + deviceTypeI915,
+				namespace + "/" + deviceTypeXe,
+			})
 		if err != nil {
 			klog.Errorf("Failed to create resource manager: %+v", err)
 			return nil
@@ -345,13 +355,20 @@ func (dp *devicePlugin) GetPreferredAllocation(rqt *pluginapi.PreferredAllocatio
 func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
 	defer dp.scanTicker.Stop()
 
-	klog.V(1).Infof("GPU '%s' resource share count = %d", deviceType, dp.options.sharedDevNum)
+	klog.V(1).Infof("GPU (%s/%s) resource share count = %d", deviceTypeI915, deviceTypeXe, dp.options.sharedDevNum)
 
-	previousCount := map[string]int{deviceType: 0, monitorType: 0}
+	previousCount := map[string]int{
+		deviceTypeI915: 0, deviceTypeXe: 0,
+		deviceTypeXe + monitorSuffix:   0,
+		deviceTypeI915 + monitorSuffix: 0}
 
 	for {
 		devTree, err := dp.scan()
 		if err != nil {
+			if errors.Is(err, rmWithMultipleDriversErr{}) {
+				return err
+			}
+
 			klog.Warning("Failed to scan: ", err)
 		}
 
@@ -426,81 +443,116 @@ func (dp *devicePlugin) devSpecForDrmFile(drmFile string) (devSpec pluginapi.Dev
 	return
 }
 
+func (dp *devicePlugin) filterOutInvalidCards(files []fs.DirEntry) []fs.DirEntry {
+	filtered := []fs.DirEntry{}
+
+	for _, f := range files {
+		if !dp.isCompatibleDevice(f.Name()) {
+			continue
+		}
+
+		_, err := os.Stat(path.Join(dp.sysfsDir, f.Name(), "device/drm"))
+		if err != nil {
+			continue
+		}
+
+		filtered = append(filtered, f)
+	}
+
+	return filtered
+}
+
+func (dp *devicePlugin) createDeviceSpecsFromDrmFiles(cardPath string) []pluginapi.DeviceSpec {
+	specs := []pluginapi.DeviceSpec{}
+
+	drmFiles, _ := os.ReadDir(path.Join(cardPath, "device/drm"))
+
+	for _, drmFile := range drmFiles {
+		devSpec, devPath, devSpecErr := dp.devSpecForDrmFile(drmFile.Name())
+		if devSpecErr != nil {
+			continue
+		}
+
+		klog.V(4).Infof("Adding %s to GPU %s", devPath, filepath.Base(cardPath))
+
+		specs = append(specs, devSpec)
+	}
+
+	return specs
+}
+
 func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 	files, err := os.ReadDir(dp.sysfsDir)
 	if err != nil {
 		return nil, errors.Wrap(err, "Can't read sysfs folder")
 	}
 
-	var monitor []pluginapi.DeviceSpec
+	monitor := make(map[string][]pluginapi.DeviceSpec, 0)
 
 	devTree := dpapi.NewDeviceTree()
 	rmDevInfos := rm.NewDeviceInfoMap()
-	tileCounts := []uint64{}
+	devProps := newDeviceProperties()
 
-	for _, f := range files {
-		var nodes []pluginapi.DeviceSpec
+	for _, f := range dp.filterOutInvalidCards(files) {
+		name := f.Name()
+		cardPath := path.Join(dp.sysfsDir, name)
 
-		if !dp.isCompatibleDevice(f.Name()) {
+		devProps.fetch(cardPath)
+
+		if devProps.isPfWithVfs {
 			continue
 		}
 
-		cardPath := path.Join(dp.sysfsDir, f.Name())
+		devSpecs := dp.createDeviceSpecsFromDrmFiles(cardPath)
 
-		drmFiles, err := os.ReadDir(path.Join(cardPath, "device/drm"))
-		if err != nil {
-			return nil, errors.Wrap(err, "Can't read device folder")
+		if len(devSpecs) == 0 {
+			continue
 		}
 
-		isPFwithVFs := pluginutils.IsSriovPFwithVFs(path.Join(dp.sysfsDir, f.Name()))
-		tileCounts = append(tileCounts, labeler.GetTileCount(dp.sysfsDir, f.Name()))
-
-		for _, drmFile := range drmFiles {
-			devSpec, devPath, devSpecErr := dp.devSpecForDrmFile(drmFile.Name())
-			if devSpecErr != nil {
-				continue
-			}
-
-			if !isPFwithVFs {
-				klog.V(4).Infof("Adding %s to GPU %s", devPath, f.Name())
+		mounts := []pluginapi.Mount{}
+		if dp.bypathFound {
+			mounts = dp.bypathMountsForPci(cardPath, name, dp.bypathDir)
+		}
 
-				nodes = append(nodes, devSpec)
-			}
+		deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devSpecs, mounts, nil, nil)
 
-			if dp.options.enableMonitoring {
-				klog.V(4).Infof("Adding %s to GPU %s/%s", devPath, monitorType, monitorID)
+		for i := 0; i < dp.options.sharedDevNum; i++ {
+			devID := fmt.Sprintf("%s-%d", name, i)
+			devTree.AddDevice(devProps.driver(), devID, deviceInfo)
 
-				monitor = append(monitor, devSpec)
-			}
+			rmDevInfos[devID] = rm.NewDeviceInfo(devSpecs, mounts, nil)
 		}
 
-		if len(nodes) > 0 {
-			mounts := []pluginapi.Mount{}
-			if dp.bypathFound {
-				mounts = dp.bypathMountsForPci(cardPath, f.Name(), dp.bypathDir)
-			}
-
-			deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, mounts, nil, nil)
-
-			for i := 0; i < dp.options.sharedDevNum; i++ {
-				devID := fmt.Sprintf("%s-%d", f.Name(), i)
-				// Currently only one device type (i915) is supported.
-				// TODO: check model ID to differentiate device models.
-				devTree.AddDevice(deviceType, devID, deviceInfo)
+		if dp.options.enableMonitoring {
+			res := devProps.monitorResource()
+			klog.V(4).Infof("For %s/%s, adding nodes: %+v", res, monitorID, devSpecs)
 
-				rmDevInfos[devID] = rm.NewDeviceInfo(nodes, mounts, nil)
-			}
+			monitor[res] = append(monitor[res], devSpecs...)
 		}
 	}
-	// all Intel GPUs are under single monitoring resource
+
+	// all Intel GPUs are under single monitoring resource per KMD
 	if len(monitor) > 0 {
-		deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, monitor, nil, nil, nil)
-		devTree.AddDevice(monitorType, monitorID, deviceInfo)
+		for resourceName, devices := range monitor {
+			deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devices, nil, nil, nil)
+			devTree.AddDevice(resourceName, monitorID, deviceInfo)
+		}
 	}
 
 	if dp.resMan != nil {
-		dp.resMan.SetDevInfos(rmDevInfos)
-		dp.resMan.SetTileCountPerCard(tileCounts)
+		if devProps.drmDriverCount() <= 1 {
+			dp.resMan.SetDevInfos(rmDevInfos)
+
+			if tileCount, err := devProps.maxTileCount(); err == nil {
+				dp.resMan.SetTileCountPerCard(tileCount)
+			}
+		} else {
+			klog.Warning("Plugin with RM doesn't support multiple DRM drivers:", devProps.drmDrivers)
+
+			err := rmWithMultipleDriversErr{}
+
+			return nil, err
+		}
 	}
 
 	return devTree, nil
@@ -521,7 +573,7 @@ func main() {
 	)
 
 	flag.StringVar(&prefix, "prefix", "", "Prefix for devfs & sysfs paths")
-	flag.BoolVar(&opts.enableMonitoring, "enable-monitoring", false, "whether to enable 'i915_monitoring' (= all GPUs) resource")
+	flag.BoolVar(&opts.enableMonitoring, "enable-monitoring", false, "whether to enable '*_monitoring' (= all GPUs) resource")
 	flag.BoolVar(&opts.resourceManagement, "resource-manager", false, "fractional GPU resource management")
 	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device")
 	flag.StringVar(&opts.preferredAllocationPolicy, "allocation-policy", "none", "modes of allocating GPU devices: balanced, packed and none")