Skip to content

Rework how the cr works + switch to k3d for tests #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gezb merged 1 commit into from
Jan 28, 2026
Merged

Rework how the cr works + switch to k3d for tests #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/lint.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
name: Lint

on:
push:
pull_request:
branches: [main]

jobs:
lint:
Expand Down
16 changes: 7 additions & 9 deletions .github/workflows/test-e2e.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
name: E2E Tests

on:
push:
pull_request:
branches: [main]

jobs:
test-e2e:
Expand All @@ -16,18 +17,15 @@ jobs:
with:
go-version-file: go.mod

- name: Install the latest version of kind
- name: Install the latest version of k3d
run: |
curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
chmod +x ./kind
sudo mv ./kind /usr/local/bin/kind

- name: Verify kind installation
run: kind version
go install github.com/k3d-io/k3d/v5@v5.8.3

- name: Verify k3d installation
run: k3d version

- name: Run Mage
uses: magefile/mage-action@v3
with:
version: latest
args: teste2e
args: teste2e
3 changes: 2 additions & 1 deletion .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
name: Tests

on:
push:
pull_request:
branches: [main]

jobs:
test:
Expand Down
7 changes: 4 additions & 3 deletions api/v1/nodedrain_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,16 @@ type NamespaceAndName struct {
type NodeDrainSpec struct {
// NodeName is the name of the node to drain
NodeName string `json:"nodeName"`
// VersionToDrainRegex is a regex to match the expected kubernetes version that we want to Drain
VersionToDrainRegex string `json:"versionToDrainRegex"`
// NodeRole is the nodes expected "role" label
NodeRole string `json:"nodeRole"`
// DisableCordon stop the controller cordoning the node
// +kubebuilder:validation:Optional
DisableCordon bool `json:"disableCordon"`
// WaitForPods waits for the evicted pods to be running again before completing
// +kubebuilder:validation:Optional
WaitForPodsToRestart bool `json:"waitForPodsToRestart"`
// IgnoreVersion ignores different kubelet versions when considering if a node group is empty (for testing)
// +kubebuilder:validation:Optional
IgnoreVersion bool `json:"ignoreVersion"`
}

// NodeDrainStatus defines the observed state of NodeDrain.
Expand Down
13 changes: 9 additions & 4 deletions config/crd/bases/k8s.gezb.co.uk_nodedrains.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,24 @@ spec:
disableCordon:
description: DisableCordon stop the controller cordoning the node
type: boolean
ignoreVersion:
description: IgnoreVersion ignores different kubelet versions when
considering if a node group is empty (for testing)
type: boolean
nodeName:
description: NodeName is the name of the node to drain
type: string
nodeRole:
description: NodeRole is the nodes expected "role" label
type: string
versionToDrainRegex:
description: VersionToDrainRegex is a regex to match the expected
kubernetes version that we want to Drain
type: string
waitForPodsToRestart:
description: WaitForPods waits for the evicted pods to be running
again before completing
type: boolean
required:
- nodeName
- nodeRole
- versionToDrainRegex
type: object
status:
description: NodeDrainStatus defines the observed state of NodeDrain.
Expand Down
21 changes: 9 additions & 12 deletions internal/controller/nodedrain_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ func (r *NodeDrainReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
return ctrl.Result{}, nil
}

// create a drain Helper
drainer, err := createDrainer(ctx, r.MgrConfig)
if err != nil {
Expand Down Expand Up @@ -170,7 +171,6 @@ func (r *NodeDrainReconciler) reconcilePending(ctx context.Context, drainer *dra
if err != nil {
if apiErrors.IsNotFound(err) {
r.logger.Error(err, "Didn't find a node matching the NodeName field", "NodeName", nodeName)
// node drain has failed - nodeName doesn't match an existing node
message := fmt.Sprintf("Node: %s not found", nodeName)
publishEvent(r.Recorder, nodeDrain, corev1.EventTypeWarning, events.EventReasonNodeNotFound, message)
setStatus(r.Recorder, nodeDrain, gezbcoukalphav1.NodeDrainPhaseFailed)
Expand All @@ -192,6 +192,7 @@ func (r *NodeDrainReconciler) reconcilePending(ctx context.Context, drainer *dra
}
}
}

podlist, err := drainer.Client.CoreV1().Pods(metav1.NamespaceAll).List(ctx,
metav1.ListOptions{
FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeDrain.Spec.NodeName}).String(),
Expand Down Expand Up @@ -232,14 +233,7 @@ func (r *NodeDrainReconciler) reconcileCordoned(ctx context.Context, drainer *dr
return true, true, nil
}

node, err := r.fetchNode(ctx, drainer, nodeDrain.Spec.NodeName)
if err != nil {
return false, false, err
}
kubeletVersion := node.Status.NodeInfo.KubeletVersion
nodeRole := node.Labels["role"]

allNodesCordoned, err := r.areAllNodesCordoned(ctx, nodeDrain.Spec.IgnoreVersion, kubeletVersion, nodeRole)
allNodesCordoned, err := r.areAllNodesCordoned(ctx, nodeDrain)
if err != nil {
return true, false, err
}
Expand Down Expand Up @@ -359,16 +353,19 @@ func (r *NodeDrainReconciler) fetchNode(ctx context.Context, drainer *drain.Help
return node, nil
}

func (r *NodeDrainReconciler) areAllNodesCordoned(ctx context.Context, ignoreVersion bool, kubeletVersion string, nodeRole string) (bool, error) {
func (r *NodeDrainReconciler) areAllNodesCordoned(ctx context.Context, nodeDrain *gezbcoukalphav1.NodeDrain) (bool, error) {
nodeList := &corev1.NodeList{}
err := r.Client.List(ctx, nodeList, &client.ListOptions{})
if err != nil {
return false, err
}

pattern := regexp.MustCompile(nodeDrain.Spec.VersionToDrainRegex)
unschedulable := true
for _, node := range nodeList.Items {
if node.Labels["role"] == nodeRole &&
(ignoreVersion || (node.Status.NodeInfo.KubeletVersion == kubeletVersion)) {
if node.Name != nodeDrain.Spec.NodeName && // ignore ourselves
node.Labels["role"] == nodeDrain.Spec.NodeRole &&
pattern.MatchString(node.Status.NodeInfo.KubeletVersion) {
if !node.Spec.Unschedulable {
unschedulable = false
}
Expand Down
7 changes: 7 additions & 0 deletions internal/controller/nodedrain_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,8 @@ func getTestNodeDrain(disableCordon bool, waitForRestart bool) *gezbcoukalphav1.
},
Spec: gezbcoukalphav1.NodeDrainSpec{
NodeName: "node1",
VersionToDrainRegex: "1.35.*",
NodeRole: "test-role",
DisableCordon: disableCordon,
WaitForPodsToRestart: waitForRestart,
},
Expand Down Expand Up @@ -550,6 +552,11 @@ func getTestNode(nodeName string, unschedulable bool) *corev1.Node {
Spec: corev1.NodeSpec{
Unschedulable: unschedulable,
},
Status: corev1.NodeStatus{
NodeInfo: corev1.NodeSystemInfo{
KubeletVersion: "1.35.5-eks",
},
},
}
}

Expand Down
24 changes: 21 additions & 3 deletions magefile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,32 @@ import (
"github.com/magefile/mage/sh"
)

const (
K3S_IMAGE = "rancher/k3s:v1.34.3-k3s1"
CLUSTER_NAME = "nd-e2e"
AGENTS = "3"
)

// Runs the End-to-End Test in a local Kind Cluster
func TestE2E() error {
err := sh.RunV("kind", "create", "cluster", "--config", "kind-config-e2e.yaml")
env := map[string]string{
"KUBECONFIG": "/tmp/nd-e2e-kubeconfig.yaml",
CLUSTER_NAME: CLUSTER_NAME,
}
k3dOptions := []string{"cluster", "create", CLUSTER_NAME, "--no-lb", "--image", K3S_IMAGE, "--agents", AGENTS}
k3dOptions = append(k3dOptions, "--k3s-arg", "--disable=traefik,servicelb,metrics-serve@server:0")
k3dOptions = append(k3dOptions, "--wait")
err := sh.RunWithV(env, "k3d", k3dOptions...)
if err != nil {
return err
}

err = sh.RunWithV(env, "go", "test", "./test/e2e/", "-v", "-ginkgo.v")
if err != nil {
return err
}

// make sure we delete the kind cluster
defer sh.RunV("kind", "delete", "cluster")
return sh.RunV("make", "test-e2e")
defer sh.RunWithV(env, "k3d", "cluster", "delete", CLUSTER_NAME)
return nil
}
7 changes: 0 additions & 7 deletions samples/nodedrain.yaml
View file
Open in desktop

This file was deleted.

30 changes: 25 additions & 5 deletions test/e2e/e2e_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,20 @@ import (
// k8sRole is the role of the node reserved for drain testing
const k8sRole = "draintest"

// k8sCuster is the name of the k3d cluster used for e2e testing
const k8sCuster = "nd-e2e"

// K8sVersionRegex is the regex to match the k8s version used in the e2e tests
const K8sVersionRegex = "^v1\\.34\\..*$"

// k8sKubeconfig is the path to the kubeconfig file used for e2e testing
const k8sKubeconfig = "/tmp/nd-e2e-kubeconfig.yaml"

// worker2Node is the name of the node reserved for drain testing
const worker2Node = "kind-worker2"
const worker2Node = "k3d-nd-e2e-agent-1"

// worker3Node is the name of the node reserved for further testing
const worker3Node = "kind-worker3"
const worker3Node = "k3d-nd-e2e-agent-2"

var (
// Optional Environment Variables:
Expand Down Expand Up @@ -51,23 +60,31 @@ func TestE2E(t *testing.T) {
}

var _ = BeforeSuite(func() {
By("Setting KUBECONFIG environment variable")
err := os.Setenv("KUBECONFIG", k8sKubeconfig)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to set KUBECONFIG environment variable")

By("Labeling and cordening the test nodes so only our workloads run on them")
// worker2
utils.LabelNode(worker2Node, k8sRole)
utils.CordonNode(worker2Node)
// worker 3
utils.LabelNode(worker3Node, k8sRole)
utils.CordonNode(worker3Node)

By("Draining the test nodes to ensure a clean state")
utils.DrainNodes([]string{worker2Node, worker3Node})

By("Ensure that Prometheus is enabled")
_ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#")

By("building the manager(Operator) image")
cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))
_, err := utils.Run(cmd)
_, err = utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image")

By("loading the manager(Operator) image on Kind")
err = utils.LoadImageToKindClusterWithName(projectImage)
By("loading the manager(Operator) image on k3d")
err = utils.LoadImageToK3dClusterWithName(k8sCuster, projectImage)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind")

// The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing.
Expand Down Expand Up @@ -106,4 +123,7 @@ var _ = AfterSuite(func() {
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n")
utils.UninstallCertManager()
}
By("Unsetting KUBECONFIG environment variable")
err := os.Unsetenv("KUBECONFIG")
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to unset KUBECONFIG environment variable")
})
Loading
Loading