Validate disk removals

.github/workflows/basic-ci.yaml

@@ -33,7 +33,7 @@ jobs:
     steps:
       - name: "Clone and check"
         uses: actions/checkout@v3
-      - name: "Build the Image for the Integration Test"
+      - name: "Build Integration Test Image and run Unit Tests"
         run: |
           BUILD_FOR_CI=true make
           ./ci/scripts/patch-ttl-repo.sh
@@ -114,7 +114,7 @@ jobs:
     steps:
       - name: "Clone and check"
         uses: actions/checkout@v3
-      - name: "Build the Image for the Integration Test"
+      - name: "Build Integration Test Image and run Unit Tests"
         run: |
           BUILD_FOR_CI=true make
           ./ci/scripts/patch-ttl-repo.sh

cmd/node-disk-manager-webhook/main.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 
+	ctrllh "github.com/harvester/harvester/pkg/generated/controllers/longhorn.io"
+	lhv1beta2 "github.com/harvester/harvester/pkg/generated/controllers/longhorn.io/v1beta2"
 	"github.com/harvester/webhook/pkg/config"
 	"github.com/harvester/webhook/pkg/server"
 	"github.com/harvester/webhook/pkg/server/admission"
@@ -28,10 +30,14 @@ import (
 const webhookName = "harvester-node-disk-manager-webhook"
 
 type resourceCaches struct {
-	bdCache           ctldiskv1.BlockDeviceCache
-	lvmVGCache        ctldiskv1.LVMVolumeGroupCache
-	storageClassCache ctlstoragev1.StorageClassCache
-	pvCache           ctlcorev1.PersistentVolumeCache
+	bdCache             ctldiskv1.BlockDeviceCache
+	lvmVGCache          ctldiskv1.LVMVolumeGroupCache
+	storageClassCache   ctlstoragev1.StorageClassCache
+	pvCache             ctlcorev1.PersistentVolumeCache
+	lhVolumeCache       lhv1beta2.VolumeCache
+	lhBackingImageCache lhv1beta2.BackingImageCache
+	lhNodeCache         lhv1beta2.NodeCache
+	lhReplicaCache      lhv1beta2.ReplicaCache
 }
 
 func main() {
@@ -117,7 +123,8 @@ func runWebhookServer(ctx context.Context, cfg *rest.Config, options *config.Opt
 		bdMutator,
 	}
 
-	bdValidator := blockdevice.NewBlockdeviceValidator(resourceCaches.bdCache, resourceCaches.storageClassCache, resourceCaches.pvCache)
+	bdValidator := blockdevice.NewBlockdeviceValidator(resourceCaches.bdCache, resourceCaches.storageClassCache, resourceCaches.pvCache,
+		resourceCaches.lhVolumeCache, resourceCaches.lhBackingImageCache, resourceCaches.lhNodeCache, resourceCaches.lhReplicaCache)
 	scValidator := storageclass.NewStorageClassValidator(resourceCaches.lvmVGCache)
 	var validators = []admission.Validator{
 		bdValidator,
@@ -156,12 +163,21 @@ func newCaches(ctx context.Context, cfg *rest.Config, threadiness int) (*resourc
 	if err != nil {
 		return nil, err
 	}
-	starters = append(starters, disks, storageFactory, coreFactory)
+	lhFactory, err := ctrllh.NewFactoryFromConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	starters = append(starters, disks, storageFactory, coreFactory, lhFactory)
 	resourceCaches := &resourceCaches{
-		bdCache:           disks.Harvesterhci().V1beta1().BlockDevice().Cache(),
-		lvmVGCache:        disks.Harvesterhci().V1beta1().LVMVolumeGroup().Cache(),
-		storageClassCache: storageFactory.Storage().V1().StorageClass().Cache(),
-		pvCache:           coreFactory.Core().V1().PersistentVolume().Cache(),
+		bdCache:             disks.Harvesterhci().V1beta1().BlockDevice().Cache(),
+		lvmVGCache:          disks.Harvesterhci().V1beta1().LVMVolumeGroup().Cache(),
+		storageClassCache:   storageFactory.Storage().V1().StorageClass().Cache(),
+		pvCache:             coreFactory.Core().V1().PersistentVolume().Cache(),
+		lhVolumeCache:       lhFactory.Longhorn().V1beta2().Volume().Cache(),
+		lhBackingImageCache: lhFactory.Longhorn().V1beta2().BackingImage().Cache(),
+		lhNodeCache:         lhFactory.Longhorn().V1beta2().Node().Cache(),
+		lhReplicaCache:      lhFactory.Longhorn().V1beta2().Replica().Cache(),
 	}
 
 	if err := start.All(ctx, threadiness, starters...); err != nil {

deploy/charts/harvester-node-disk-manager/templates/rbac.yaml

@@ -68,6 +68,9 @@ rules:
   - apiGroups: [ "admissionregistration.k8s.io" ]
     resources: [ "validatingwebhookconfigurations", "mutatingwebhookconfigurations" ]
     verbs: [ "*" ]
+  - apiGroups: ["longhorn.io"]
+    resources: [ "volumes", "nodes", "backingimages", "replicas" ] 
+    verbs: [ "get", "watch", "list" ]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

pkg/utils/fake/backingimage.go

@@ -0,0 +1,33 @@
+package fake
+
+import (
+	lhv1beta2 "github.com/harvester/harvester/pkg/generated/controllers/longhorn.io/v1beta2"
+	lhv1 "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+type FakeBackingImageCache struct {
+	backingImages []*lhv1.BackingImage
+}
+
+func NewBackingImageCache(backingImagesToServe []*lhv1.BackingImage) lhv1beta2.BackingImageCache {
+	return &FakeBackingImageCache{
+		backingImages: backingImagesToServe,
+	}
+}
+
+func (f *FakeBackingImageCache) AddIndexer(indexName string, indexer generic.Indexer[*lhv1.BackingImage]) {
+}
+
+func (f *FakeBackingImageCache) Get(namespace string, name string) (*lhv1.BackingImage, error) {
+	panic("unimplemented")
+}
+
+func (f *FakeBackingImageCache) GetByIndex(indexName string, key string) ([]*lhv1.BackingImage, error) {
+	return f.backingImages, nil
+}
+
+func (f *FakeBackingImageCache) List(namespace string, selector labels.Selector) ([]*lhv1.BackingImage, error) {
+	panic("unimplemented")
+}

pkg/utils/fake/blockdevice.go

@@ -0,0 +1,52 @@
+package fake
+
+import (
+	diskv1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1"
+	ctldiskv1 "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io/v1beta1"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type FakeBlockDeviceCache struct {
+	devices []*diskv1.BlockDevice
+}
+
+func NewBlockDeviceCache(devicesToServe []*diskv1.BlockDevice) ctldiskv1.BlockDeviceCache {
+	return &FakeBlockDeviceCache{
+		devices: devicesToServe,
+	}
+}
+
+func (c *FakeBlockDeviceCache) AddIndexer(indexName string, indexer generic.Indexer[*diskv1.BlockDevice]) {
+	panic("unimplemented")
+}
+
+func (c *FakeBlockDeviceCache) Get(namespace, name string) (*diskv1.BlockDevice, error) {
+	for _, device := range c.devices {
+		if device.Namespace == namespace && device.Name == name {
+			return device.DeepCopy(), nil
+		}
+	}
+	return nil, errors.NewNotFound(schema.GroupResource{}, name)
+}
+
+func (c *FakeBlockDeviceCache) GetByIndex(indexName, key string) ([]*diskv1.BlockDevice, error) {
+	panic("unimplemented")
+}
+
+func (c *FakeBlockDeviceCache) List(namespace string, selector labels.Selector) ([]*diskv1.BlockDevice, error) {
+	var matching []*diskv1.BlockDevice
+
+	for _, device := range c.devices {
+		if namespace != "" && device.Namespace != namespace {
+			continue
+		}
+
+		if selector.Matches(labels.Set(device.GetLabels())) {
+			matching = append(matching, device.DeepCopy())
+		}
+	}
+	return matching, nil
+}

pkg/utils/fake/longhornnode.go

@@ -0,0 +1,33 @@
+package fake
+
+import (
+	lhv1beta2 "github.com/harvester/harvester/pkg/generated/controllers/longhorn.io/v1beta2"
+	lhv1 "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+type FakeLonghornNodeCache struct {
+	nodes []*lhv1.Node
+}
+
+func NewLonghornNodeCache(nodesToServe []*lhv1.Node) lhv1beta2.NodeCache {
+	return &FakeLonghornNodeCache{
+		nodes: nodesToServe,
+	}
+}
+
+func (f *FakeLonghornNodeCache) AddIndexer(indexName string, indexer generic.Indexer[*lhv1.Node]) {
+}
+
+func (f *FakeLonghornNodeCache) Get(namespace string, name string) (*lhv1.Node, error) {
+	return f.nodes[0], nil
+}
+
+func (f *FakeLonghornNodeCache) GetByIndex(indexName string, key string) ([]*lhv1.Node, error) {
+	return f.nodes, nil
+}
+
+func (f *FakeLonghornNodeCache) List(namespace string, selector labels.Selector) ([]*lhv1.Node, error) {
+	panic("unimplemented")
+}

pkg/utils/fake/persistentvolumecache.go

@@ -0,0 +1,47 @@
+package fake
+
+import (
+	ctlcorev1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type FakePersistentVolumeCache struct {
+	pvs []*v1.PersistentVolume
+}
+
+func NewPersistentVolumeCache(pvsToServe []*v1.PersistentVolume) ctlcorev1.PersistentVolumeCache {
+	return &FakePersistentVolumeCache{
+		pvs: pvsToServe,
+	}
+}
+
+func (f *FakePersistentVolumeCache) AddIndexer(indexName string, indexer generic.Indexer[*v1.PersistentVolume]) {
+	panic("unimplemented")
+}
+
+func (f *FakePersistentVolumeCache) Get(name string) (*v1.PersistentVolume, error) {
+	for _, pv := range f.pvs {
+		if pv.Name == name {
+			return pv.DeepCopy(), nil
+		}
+	}
+	return nil, errors.NewNotFound(schema.GroupResource{}, name)
+}
+
+func (f *FakePersistentVolumeCache) GetByIndex(indexName string, key string) ([]*v1.PersistentVolume, error) {
+	panic("unimplemented")
+}
+
+func (f *FakePersistentVolumeCache) List(selector labels.Selector) ([]*v1.PersistentVolume, error) {
+	var matchingPVs []*v1.PersistentVolume
+	for _, pv := range f.pvs {
+		if selector.Matches(labels.Set(pv.Labels)) {
+			matchingPVs = append(matchingPVs, pv.DeepCopy())
+		}
+	}
+	return matchingPVs, nil
+}

pkg/utils/fake/replicacache.go

@@ -0,0 +1,33 @@
+package fake
+
+import (
+	lhv1beta2 "github.com/harvester/harvester/pkg/generated/controllers/longhorn.io/v1beta2"
+	lhv1 "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+type FakeReplicaCache struct {
+	replicas []*lhv1.Replica
+}
+
+func NewReplicaCache(replicasToServe []*lhv1.Replica) lhv1beta2.ReplicaCache {
+	return &FakeReplicaCache{
+		replicas: replicasToServe,
+	}
+}
+
+func (f *FakeReplicaCache) AddIndexer(indexName string, indexer generic.Indexer[*lhv1.Replica]) {
+}
+
+func (f *FakeReplicaCache) Get(namespace string, name string) (*lhv1.Replica, error) {
+	panic("unimplemented")
+}
+
+func (f *FakeReplicaCache) GetByIndex(indexName string, key string) ([]*lhv1.Replica, error) {
+	return f.replicas, nil
+}
+
+func (f *FakeReplicaCache) List(namespace string, selector labels.Selector) ([]*lhv1.Replica, error) {
+	panic("unimplemented")
+}

pkg/utils/fake/storageclass.go

@@ -0,0 +1,48 @@
+package fake
+
+import (
+	ctlstoragev1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/storage/v1"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type FakeStorageClassCache struct {
+	scs []*storagev1.StorageClass
+}
+
+func NewStorageClassCache(scsToServe []*storagev1.StorageClass) ctlstoragev1.StorageClassCache {
+	return &FakeStorageClassCache{
+		scs: scsToServe,
+	}
+}
+
+func (f *FakeStorageClassCache) AddIndexer(indexName string, indexer generic.Indexer[*storagev1.StorageClass]) {
+	panic("unimplemented")
+}
+
+func (f *FakeStorageClassCache) Get(name string) (*storagev1.StorageClass, error) {
+	for _, storageClass := range f.scs {
+		if storageClass.Name == name {
+			return storageClass.DeepCopy(), nil
+		}
+	}
+	return nil, errors.NewNotFound(schema.GroupResource{}, name)
+}
+
+func (f *FakeStorageClassCache) GetByIndex(indexName string, key string) ([]*storagev1.StorageClass, error) {
+	panic("unimplemented")
+}
+
+func (f *FakeStorageClassCache) List(selector labels.Selector) ([]*storagev1.StorageClass, error) {
+	var matchingSCs []*storagev1.StorageClass
+	for _, storageClass := range f.scs {
+		// Check if the StorageClass's labels match the provided selector.
+		if selector.Matches(labels.Set(storageClass.Labels)) {
+			matchingSCs = append(matchingSCs, storageClass.DeepCopy())
+		}
+	}
+	return matchingSCs, nil
+}

pkg/utils/fake/volumecache.go

@@ -0,0 +1,50 @@
+package fake
+
+import (
+	lhv1 "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
+	"github.com/rancher/wrangler/v3/pkg/generic"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	lhv1beta2 "github.com/harvester/harvester/pkg/generated/controllers/longhorn.io/v1beta2"
+)
+
+type FakeVolumeCache struct {
+	volumes []*lhv1.Volume
+}
+
+func NewVolumeCache(volsToServe []*lhv1.Volume) lhv1beta2.VolumeCache {
+	return &FakeVolumeCache{
+		volumes: volsToServe,
+	}
+}
+
+func (f *FakeVolumeCache) AddIndexer(indexName string, indexer generic.Indexer[*lhv1.Volume]) {
+	panic("unimplemented")
+}
+
+func (f *FakeVolumeCache) Get(namespace string, name string) (*lhv1.Volume, error) {
+	for _, volume := range f.volumes {
+		if volume.Namespace == namespace && volume.Name == name {
+			return volume.DeepCopy(), nil
+		}
+	}
+	return nil, errors.NewNotFound(schema.GroupResource{}, name)
+}
+
+func (f *FakeVolumeCache) GetByIndex(indexName string, key string) ([]*lhv1.Volume, error) {
+	panic("unimplemented")
+}
+
+func (f *FakeVolumeCache) List(namespace string, selector labels.Selector) ([]*lhv1.Volume, error) {
+	var matchingVolumes []*lhv1.Volume
+	for _, volume := range f.volumes {
+		if namespace == "" || volume.Namespace == namespace {
+			if selector.Matches(labels.Set(volume.Labels)) {
+				matchingVolumes = append(matchingVolumes, volume.DeepCopy())
+			}
+		}
+	}
+	return matchingVolumes, nil
+}