Add custom PCI DevicePlugin

>
Co-authored-by: Gaurav Mehta <[email protected]>

Author: Tobi Lehman

.gitignore

@@ -2,4 +2,5 @@
 pcidevices
 bin/
 dist/
-.idea
+.idea
+TAGS

Makefile

@@ -1,5 +1,8 @@
 TARGETS := $(shell ls scripts)
 
+tags:
+	gotags -R pkg > TAGS
+
 .dapper:
 	@echo Downloading dapper
 	@curl -sL https://releases.rancher.com/dapper/latest/dapper-`uname -s`-`uname -m` > .dapper.tmp

README.md

@@ -14,6 +14,9 @@ This operator introduces these CRDs:
 - PCIDevice
 - PCIDeviceClaim
 
+It also introduces a custom PCIDevicePlugin. The way the deviceplugin works is by storing all 
+PCIDevices with the same resourceName. Then when one is claimed, the deviceplugin marks that device state as "healthy".
+
 ## PCIDevice
 
 This custom resource represents PCI Devices on the host. 
@@ -131,7 +134,7 @@ This only detects the presence or absence of device, not the number of them.
 Another reason not to use these simple labels is that we want to be able to allow our customers to set custom RBAC rules that restrict who can use which device in the cluster. We can do that with a custom `PCIDevice` CRD, but it's not clear how to do that with node labels.
 
 ## License
-Copyright (c) 2022 [Rancher Labs, Inc.](http://rancher.com)
+Copyright (c) 2023 [Rancher Labs, Inc.](http://rancher.com)
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -143,4 +146,4 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License.
+limitations under the License.

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	github.com/evanphx/json-patch v5.6.0+incompatible
+	github.com/fsnotify/fsnotify v1.5.1
 	github.com/gorilla/mux v1.8.0
 	github.com/harvester/harvester v0.0.2-0.20220916012220-3bcba9d3747f
 	github.com/jaypipes/ghw v0.9.0
@@ -18,6 +19,7 @@ require (
 	github.com/u-root/u-root v7.0.0+incompatible
 	github.com/urfave/cli/v2 v2.11.1
 	github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54
+	google.golang.org/grpc v1.47.0
 	k8s.io/api v0.24.3
 	k8s.io/apimachinery v0.24.3
 	k8s.io/client-go v12.0.0+incompatible
@@ -29,7 +31,6 @@ require (
 
 require (
 	github.com/coreos/prometheus-operator v0.38.1-0.20200424145508-7e176fda06cc // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-kit/kit v0.9.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -44,7 +45,6 @@ require (
 	github.com/openshift/client-go v0.0.0 // indirect
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
 	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
-	google.golang.org/grpc v1.47.0 // indirect
 	howett.net/plist v1.0.0 // indirect
 )
 
@@ -54,7 +54,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful v2.15.0+incompatible // indirect
+	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/zapr v1.2.3 // indirect
@@ -92,9 +92,9 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
-	golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 // indirect
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced // indirect
+	golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced
 	golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7 // indirect
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 // indirect
@@ -116,7 +116,7 @@ require (
 	k8s.io/kubernetes v1.25.1
 	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
 	kubevirt.io/api v0.54.0
-	kubevirt.io/containerized-data-importer-api v1.47.0 // indirect
+	kubevirt.io/containerized-data-importer-api v1.50.0 // indirect
 	kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect

go.sum

@@ -932,8 +932,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 h1:S25/rfnfsMVgORT4/J61MJ7rdyseOZOyvLIrZEZ7s6s=
-golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

manifests/pcidevices-operator.yaml

@@ -90,13 +90,19 @@ spec:
               name: modules
             - mountPath: /sys
               name: sys
+            - mountPath: /var/lib/kubelet/device-plugins
+              name: device-plugins
       priorityClassName: system-node-critical
       serviceAccountName: pcidevices
       volumes:
         - hostPath:
             path: /lib/modules
             type: Directory
           name: modules
+        - hostPath:
+            path: /var/lib/kubelet/device-plugins
+            type: Directory
+          name: device-plugins
         - hostPath:
             path: /sys
             type: Directory

pkg/apis/devices.harvesterhci.io/v1beta1/pcidevice.go

@@ -2,18 +2,16 @@ package v1beta1
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"regexp"
 
 	"github.com/jaypipes/ghw/pkg/pci"
 	"github.com/jaypipes/ghw/pkg/util"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	device_manager "kubevirt.io/kubevirt/pkg/virt-handler/device-manager"
 )
 
-const pciBasePath = "/sys/bus/pci/devices/"
-
 // +genclient
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -112,16 +110,17 @@ func resourceName(dev *pci.Device) string {
 	return fmt.Sprintf("%s/%s", vendorCleaned, dev.Product.ID)
 }
 
-func (status *PCIDeviceStatus) Update(dev *pci.Device, hostname string) {
+func (status *PCIDeviceStatus) Update(dev *pci.Device, hostname string, iommuGroups map[string]int) {
 	status.Address = dev.Address
 	status.VendorId = dev.Vendor.ID
 	status.DeviceId = dev.Product.ID
 	status.ClassId = fmt.Sprintf("%s%s", dev.Class.ID, dev.Subclass.ID)
 	// Generate the ResourceName field, this is used by KubeVirt to schedule the VM to the node
 	status.ResourceName = resourceName(dev)
 	status.Description = description(dev)
-	if iommuGroup, err := device_manager.Handler.GetDeviceIOMMUGroup(pciBasePath, dev.Address); err == nil {
-		status.IOMMUGroup = iommuGroup
+	group, ok := iommuGroups[dev.Address]
+	if ok {
+		status.IOMMUGroup = strconv.Itoa(group)
 	}
 	status.KernelDriverInUse = dev.Driver
 	status.NodeName = hostname

pkg/controller/pcidevice/pcidevice_controller.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/harvester/pcidevices/pkg/iommu"
+
 	v1beta1 "github.com/harvester/pcidevices/pkg/apis/devices.harvesterhci.io/v1beta1"
 	ctl "github.com/harvester/pcidevices/pkg/generated/controllers/devices.harvesterhci.io/v1beta1"
 	"github.com/harvester/pcidevices/pkg/util/nichelper"
@@ -60,7 +62,12 @@ func Register(
 }
 
 func (h Handler) reconcilePCIDevices(nodename string) error {
-	// List all PCI Devices on host
+	// Build up the IOMMU group map
+	iommuGroupPaths, err := iommu.GroupPaths()
+	if err != nil {
+		return err
+	}
+	iommuGroupMap := iommu.GroupMapForPCIDevices(iommuGroupPaths)
 
 	commonLabels := map[string]string{"nodename": nodename} // label
 	var setOfRealPCIAddrs map[string]bool = make(map[string]bool)
@@ -69,9 +76,7 @@ func (h Handler) reconcilePCIDevices(nodename string) error {
 			setOfRealPCIAddrs[dev.Address] = true
 			name := v1beta1.PCIDeviceNameForHostname(dev, nodename)
 			// Check if device is stored
-			var err error
-			var devCR *v1beta1.PCIDevice
-			devCR, err = h.client.Get(name, metav1.GetOptions{})
+			devCR, err := h.client.Get(name, metav1.GetOptions{})
 
 			if err != nil {
 				if apierrors.IsNotFound(err) {
@@ -95,13 +100,14 @@ func (h Handler) reconcilePCIDevices(nodename string) error {
 
 			devCopy := devCR.DeepCopy()
 			// Update only modifies the status, no need to update the main object
-			devCopy.Status.Update(dev, nodename) // update the in-memory CR with the current PCI info
+			devCopy.Status.Update(dev, nodename, iommuGroupMap) // update the in-memory CR with the current PCI info
 			_, err = h.client.UpdateStatus(devCopy)
 			if err != nil {
 				logrus.Errorf("[PCIDeviceController] Failed to update status sub-resource: %v", err)
 				return err
 			}
 		}
+
 	}
 
 	// remove non-existent devices