diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index dd7bcff..53a0843 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,7 +1,8 @@ name: Lint on: - push: + pull_request: + branches: [main] jobs: lint: diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 5e629c2..33b235a 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,7 +1,8 @@ name: E2E Tests on: - push: + pull_request: + branches: [main] jobs: test-e2e: @@ -16,18 +17,15 @@ jobs: with: go-version-file: go.mod - - name: Install the latest version of kind + - name: Install the latest version of k3d run: | - curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind - - - name: Verify kind installation - run: kind version + go install github.com/k3d-io/k3d/v5@v5.8.3 + - name: Verify k3d installation + run: k3d version - name: Run Mage uses: magefile/mage-action@v3 with: version: latest - args: teste2e + args: teste2e \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63c38e2..fb34546 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,8 @@ name: Tests on: - push: + pull_request: + branches: [main] jobs: test: diff --git a/api/v1/nodedrain_types.go b/api/v1/nodedrain_types.go index 36c3bba..8c454d1 100644 --- a/api/v1/nodedrain_types.go +++ b/api/v1/nodedrain_types.go @@ -29,15 +29,16 @@ type NamespaceAndName struct { type NodeDrainSpec struct { // NodeName is the name of the node to drain NodeName string `json:"nodeName"` + // VersionToDrainRegex is a regex to match the expected kubernetes version that we want to Drain + VersionToDrainRegex string `json:"versionToDrainRegex"` + // NodeRole is the nodes expected "role" label + NodeRole string `json:"nodeRole"` // DisableCordon stop the controller cordoning the node // +kubebuilder:validation:Optional DisableCordon bool `json:"disableCordon"` // WaitForPods waits for the evicted pods to be running again before completing // +kubebuilder:validation:Optional WaitForPodsToRestart bool `json:"waitForPodsToRestart"` - // IgnoreVersion ignores different kubelet versions when considering if a node group is empty (for testing) - // +kubebuilder:validation:Optional - IgnoreVersion bool `json:"ignoreVersion"` } // NodeDrainStatus defines the observed state of NodeDrain. diff --git a/config/crd/bases/k8s.gezb.co.uk_nodedrains.yaml b/config/crd/bases/k8s.gezb.co.uk_nodedrains.yaml index e7ce180..02fef2c 100644 --- a/config/crd/bases/k8s.gezb.co.uk_nodedrains.yaml +++ b/config/crd/bases/k8s.gezb.co.uk_nodedrains.yaml @@ -51,19 +51,24 @@ spec: disableCordon: description: DisableCordon stop the controller cordoning the node type: boolean - ignoreVersion: - description: IgnoreVersion ignores different kubelet versions when - considering if a node group is empty (for testing) - type: boolean nodeName: description: NodeName is the name of the node to drain type: string + nodeRole: + description: NodeRole is the nodes expected "role" label + type: string + versionToDrainRegex: + description: VersionToDrainRegex is a regex to match the expected + kubernetes version that we want to Drain + type: string waitForPodsToRestart: description: WaitForPods waits for the evicted pods to be running again before completing type: boolean required: - nodeName + - nodeRole + - versionToDrainRegex type: object status: description: NodeDrainStatus defines the observed state of NodeDrain. diff --git a/internal/controller/nodedrain_controller.go b/internal/controller/nodedrain_controller.go index 8a30a21..82158c6 100644 --- a/internal/controller/nodedrain_controller.go +++ b/internal/controller/nodedrain_controller.go @@ -95,6 +95,7 @@ func (r *NodeDrainReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } return ctrl.Result{}, nil } + // create a drain Helper drainer, err := createDrainer(ctx, r.MgrConfig) if err != nil { @@ -170,7 +171,6 @@ func (r *NodeDrainReconciler) reconcilePending(ctx context.Context, drainer *dra if err != nil { if apiErrors.IsNotFound(err) { r.logger.Error(err, "Didn't find a node matching the NodeName field", "NodeName", nodeName) - // node drain has failed - nodeName doesn't match an existing node message := fmt.Sprintf("Node: %s not found", nodeName) publishEvent(r.Recorder, nodeDrain, corev1.EventTypeWarning, events.EventReasonNodeNotFound, message) setStatus(r.Recorder, nodeDrain, gezbcoukalphav1.NodeDrainPhaseFailed) @@ -192,6 +192,7 @@ func (r *NodeDrainReconciler) reconcilePending(ctx context.Context, drainer *dra } } } + podlist, err := drainer.Client.CoreV1().Pods(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeDrain.Spec.NodeName}).String(), @@ -232,14 +233,7 @@ func (r *NodeDrainReconciler) reconcileCordoned(ctx context.Context, drainer *dr return true, true, nil } - node, err := r.fetchNode(ctx, drainer, nodeDrain.Spec.NodeName) - if err != nil { - return false, false, err - } - kubeletVersion := node.Status.NodeInfo.KubeletVersion - nodeRole := node.Labels["role"] - - allNodesCordoned, err := r.areAllNodesCordoned(ctx, nodeDrain.Spec.IgnoreVersion, kubeletVersion, nodeRole) + allNodesCordoned, err := r.areAllNodesCordoned(ctx, nodeDrain) if err != nil { return true, false, err } @@ -359,16 +353,19 @@ func (r *NodeDrainReconciler) fetchNode(ctx context.Context, drainer *drain.Help return node, nil } -func (r *NodeDrainReconciler) areAllNodesCordoned(ctx context.Context, ignoreVersion bool, kubeletVersion string, nodeRole string) (bool, error) { +func (r *NodeDrainReconciler) areAllNodesCordoned(ctx context.Context, nodeDrain *gezbcoukalphav1.NodeDrain) (bool, error) { nodeList := &corev1.NodeList{} err := r.Client.List(ctx, nodeList, &client.ListOptions{}) if err != nil { return false, err } + + pattern := regexp.MustCompile(nodeDrain.Spec.VersionToDrainRegex) unschedulable := true for _, node := range nodeList.Items { - if node.Labels["role"] == nodeRole && - (ignoreVersion || (node.Status.NodeInfo.KubeletVersion == kubeletVersion)) { + if node.Name != nodeDrain.Spec.NodeName && // ignore ourselves + node.Labels["role"] == nodeDrain.Spec.NodeRole && + pattern.MatchString(node.Status.NodeInfo.KubeletVersion) { if !node.Spec.Unschedulable { unschedulable = false } diff --git a/internal/controller/nodedrain_controller_test.go b/internal/controller/nodedrain_controller_test.go index a213d3d..a58da95 100644 --- a/internal/controller/nodedrain_controller_test.go +++ b/internal/controller/nodedrain_controller_test.go @@ -521,6 +521,8 @@ func getTestNodeDrain(disableCordon bool, waitForRestart bool) *gezbcoukalphav1. }, Spec: gezbcoukalphav1.NodeDrainSpec{ NodeName: "node1", + VersionToDrainRegex: "1.35.*", + NodeRole: "test-role", DisableCordon: disableCordon, WaitForPodsToRestart: waitForRestart, }, @@ -550,6 +552,11 @@ func getTestNode(nodeName string, unschedulable bool) *corev1.Node { Spec: corev1.NodeSpec{ Unschedulable: unschedulable, }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ + KubeletVersion: "1.35.5-eks", + }, + }, } } diff --git a/magefile.go b/magefile.go index bbf08b6..a551a6e 100644 --- a/magefile.go +++ b/magefile.go @@ -7,14 +7,32 @@ import ( "github.com/magefile/mage/sh" ) +const ( + K3S_IMAGE = "rancher/k3s:v1.34.3-k3s1" + CLUSTER_NAME = "nd-e2e" + AGENTS = "3" +) + // Runs the End-to-End Test in a local Kind Cluster func TestE2E() error { - err := sh.RunV("kind", "create", "cluster", "--config", "kind-config-e2e.yaml") + env := map[string]string{ + "KUBECONFIG": "/tmp/nd-e2e-kubeconfig.yaml", + CLUSTER_NAME: CLUSTER_NAME, + } + k3dOptions := []string{"cluster", "create", CLUSTER_NAME, "--no-lb", "--image", K3S_IMAGE, "--agents", AGENTS} + k3dOptions = append(k3dOptions, "--k3s-arg", "--disable=traefik,servicelb,metrics-serve@server:0") + k3dOptions = append(k3dOptions, "--wait") + err := sh.RunWithV(env, "k3d", k3dOptions...) + if err != nil { + return err + } + + err = sh.RunWithV(env, "go", "test", "./test/e2e/", "-v", "-ginkgo.v") if err != nil { return err } // make sure we delete the kind cluster - defer sh.RunV("kind", "delete", "cluster") - return sh.RunV("make", "test-e2e") + defer sh.RunWithV(env, "k3d", "cluster", "delete", CLUSTER_NAME) + return nil } diff --git a/samples/nodedrain.yaml b/samples/nodedrain.yaml deleted file mode 100644 index a4772a7..0000000 --- a/samples/nodedrain.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: k8s.gezb.co.uk/v1 -kind: NodeDrain -metadata: - name: nodedrain-sample -spec: - nodeName: kind-worker3 - ignoreVersion: true \ No newline at end of file diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 8d4ca5f..dc48f57 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -15,11 +15,20 @@ import ( // k8sRole is the role of the node reserved for drain testing const k8sRole = "draintest" +// k8sCuster is the name of the k3d cluster used for e2e testing +const k8sCuster = "nd-e2e" + +// K8sVersionRegex is the regex to match the k8s version used in the e2e tests +const K8sVersionRegex = "^v1\\.34\\..*$" + +// k8sKubeconfig is the path to the kubeconfig file used for e2e testing +const k8sKubeconfig = "/tmp/nd-e2e-kubeconfig.yaml" + // worker2Node is the name of the node reserved for drain testing -const worker2Node = "kind-worker2" +const worker2Node = "k3d-nd-e2e-agent-1" // worker3Node is the name of the node reserved for further testing -const worker3Node = "kind-worker3" +const worker3Node = "k3d-nd-e2e-agent-2" var ( // Optional Environment Variables: @@ -51,6 +60,10 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Setting KUBECONFIG environment variable") + err := os.Setenv("KUBECONFIG", k8sKubeconfig) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to set KUBECONFIG environment variable") + By("Labeling and cordening the test nodes so only our workloads run on them") // worker2 utils.LabelNode(worker2Node, k8sRole) @@ -58,16 +71,20 @@ var _ = BeforeSuite(func() { // worker 3 utils.LabelNode(worker3Node, k8sRole) utils.CordonNode(worker3Node) + + By("Draining the test nodes to ensure a clean state") + utils.DrainNodes([]string{worker2Node, worker3Node}) + By("Ensure that Prometheus is enabled") _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") By("building the manager(Operator) image") cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage)) - _, err := utils.Run(cmd) + _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") - By("loading the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectImage) + By("loading the manager(Operator) image on k3d") + err = utils.LoadImageToK3dClusterWithName(k8sCuster, projectImage) ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing. @@ -106,4 +123,7 @@ var _ = AfterSuite(func() { _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") utils.UninstallCertManager() } + By("Unsetting KUBECONFIG environment variable") + err := os.Unsetenv("KUBECONFIG") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to unset KUBECONFIG environment variable") }) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 54316d0..0743ec7 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -225,34 +225,20 @@ var _ = Describe("Manager", Ordered, func() { By("Uncordening our test node") utils.UncordonNode(worker2Node) - By("Verifying the number of pods on the node(s) is 4") - expectNumberOfPodsRunning(4) // two nodes + By("Verifying the number of pods on the node(s) is 0") + expectNumberOfPodsRunning(0) createStatefulSetWithName("nginx") createStatefulSetWithName("blocking") By("Waiting for all pods to be running") - expectNumberOfPodsRunning(6) + expectNumberOfPodsRunning(2) - By("Creating a nodeDrain for our node") - nodeDrain := fmt.Sprintf(` -apiVersion: k8s.gezb.co.uk/v1 -kind: NodeDrain -metadata: - name: nodedrain-sample -spec: - nodeName: %s - ignoreVersion: true -`, worker2Node) - nodeDrainFile, err := utils.CreateTempFile(nodeDrain) - Expect(err).NotTo(HaveOccurred(), "Failed apply nodeDrain") - cmd := exec.Command("kubectl", "apply", "-f", nodeDrainFile) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed apply nodeDrain") + nodeDrainFile := applyNodeDrain(false) By("Waiting for all pods to be removed") - expectNumberOfPodsRunning(4) + expectNumberOfPodsRunning(0) nodeDrainIsPhaseCompleted := func(g Gomega) { cmd := exec.Command("kubectl", "get", "nodedrain", "nodedrain-sample") @@ -262,8 +248,11 @@ spec: } Eventually(nodeDrainIsPhaseCompleted, 5*time.Minute).Should(Succeed()) - cmd = exec.Command("kubectl", "delete", "-f", nodeDrainFile) - _, err = utils.Run(cmd) + By("Clean up after the test") + + By("deleting the NodeDrain") + cmd := exec.Command("kubectl", "delete", "-f", nodeDrainFile) + _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed delete nodeDrain") err = os.Remove(nodeDrainFile) Expect(err).NotTo(HaveOccurred(), "Failed remove nodeDrainFile") @@ -278,109 +267,83 @@ spec: By("re-cordening our test node") utils.CordonNode(worker2Node) - }) + }) - It("drain should be blocked by a DrainCheck until pods matching the regex are deleted", func() { + It("drain should be blocked by a DrainCheck until pods matching the regex are deleted", func() { - By("Uncordening our test node") - utils.UncordonNode(worker2Node) + By("Uncordening our test node") + utils.UncordonNode(worker2Node) - By("Verifying the number of pods on the node(s) is 2") + By("Verifying the number of pods on the node(s) is 2") - expectNumberOfPodsRunning(4) + expectNumberOfPodsRunning(0) - createStatefulSetWithName("nginx") - createStatefulSetWithName("blocking") + createStatefulSetWithName("nginx") + createStatefulSetWithName("blocking") - By("Waiting for all pods to be running") - expectNumberOfPodsRunning(6) + By("Waiting for all pods to be running") + expectNumberOfPodsRunning(2) - By("creating a DrainCheck for `blocking-.` pods") + By("creating a DrainCheck for `blocking-.` pods") - drainCheck := ` -apiVersion: k8s.gezb.co.uk/v1 -kind: DrainCheck -metadata: - name: draincheck-sample - #namespace: %s -spec: - podRegex: ^blocking-.* -` - drainCheckFile, err := utils.CreateTempFile(drainCheck) - Expect(err).NotTo(HaveOccurred(), "Failed apply "+drainCheckFile) + drainCheckFile := applyDrainCheck("^blocking-.*") - cmd := exec.Command("kubectl", "apply", "-f", drainCheckFile) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed apply DrainCheck") + nodeDrainFile := applyNodeDrain(false) - By("Creating a nodeDrain for our node") - nodeDrain := fmt.Sprintf(` -apiVersion: k8s.gezb.co.uk/v1 -kind: NodeDrain -metadata: - name: nodedrain-sample -spec: - nodeName: %s - ignoreVersion: true -`, worker2Node) + By("Checking we still have all pods running") + expectNumberOfPodsRunning(2) - nodeDrainFile, err := utils.CreateTempFile(nodeDrain) - Expect(err).NotTo(HaveOccurred(), "Failed apply nodeDrain") - cmd = exec.Command("kubectl", "apply", "-f", nodeDrainFile) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed apply nodeDrain") + nodedrainIsPhaseCordened := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "nodedrain", "nodedrain-sample") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("nodedrain-sample PodsBlockingDrain"), + "coredened NodeDrain not found") + } + Eventually(nodedrainIsPhaseCordened, 5*time.Minute).Should(Succeed()) - By("Checking we still have all pods running") - expectNumberOfPodsRunning(6) + By("Clean up after the test") - nodedrainIsPhaseCordened := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "nodedrain", "nodedrain-sample") - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("nodedrain-sample PodsBlockingDrain"), - "coredened NodeDrain not found") - } - Eventually(nodedrainIsPhaseCordened, 5*time.Minute).Should(Succeed()) + By("Deleting the blocking statefulsets") + cmd := exec.Command("kubectl", "delete", "statefulset", "blocking") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed delete nodeDrain") - By("Deleting the blocking statefulsets") - cmd = exec.Command("kubectl", "delete", "statefulset", "blocking") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed delete nodeDrain") + By("Drain should run and we should be left with no pods") + expectNumberOfPodsRunning(0) - By("Drain should run and we should be left with only deamonsets") - expectNumberOfPodsRunning(4) + nodeDrainIsPhaseCompleted := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "nodedrain", "nodedrain-sample") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("nodedrain-sample Completed")) + } + Eventually(nodeDrainIsPhaseCompleted, 5*time.Minute).Should(Succeed()) - nodeDrainIsPhaseCompleted := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "nodedrain", "nodedrain-sample") - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("nodedrain-sample Completed")) - } - Eventually(nodeDrainIsPhaseCompleted, 5*time.Minute).Should(Succeed()) + By("Clean up after the test") - By(" Deleting nginx statefulsets") - cmd = exec.Command("kubectl", "delete", "statefulset", "nginx") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed delete nginx statefulset") + By(" Deleting nginx statefulsets") + cmd = exec.Command("kubectl", "delete", "statefulset", "nginx") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed delete nginx statefulset") - By("deleting the NpdeDrain") - cmd = exec.Command("kubectl", "delete", "-f", nodeDrainFile) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed delete nodeDrain") - err = os.Remove(nodeDrainFile) - Expect(err).NotTo(HaveOccurred(), "Failed remove nodeDrainFile") + By("deleting the NpdeDrain") + cmd = exec.Command("kubectl", "delete", "-f", nodeDrainFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed delete nodeDrain") + err = os.Remove(nodeDrainFile) + Expect(err).NotTo(HaveOccurred(), "Failed remove nodeDrainFile") - By("removing the drainCheck") - cmd = exec.Command("kubectl", "delete", "-f", drainCheckFile) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed remove drainCheck") - err = os.Remove(drainCheckFile) - Expect(err).NotTo(HaveOccurred(), "Failed remove nodeDrainFile") + By("removing the drainCheck") + cmd = exec.Command("kubectl", "delete", "-f", drainCheckFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed remove drainCheck") + err = os.Remove(drainCheckFile) + Expect(err).NotTo(HaveOccurred(), "Failed remove nodeDrainFile") - By("re-cordening our test node") - utils.CordonNode(worker2Node) - }) + By("re-cordening our test node") + utils.CordonNode(worker2Node) }) It("if another node with the same 'role' & version exists uncodened should hold until it is cordened", func() { @@ -390,13 +353,13 @@ spec: By("Verifying the number of pods on the node(s)") - expectNumberOfPodsRunning(4) + expectNumberOfPodsRunning(0) createStatefulSetWithName("nginx") createStatefulSetWithName("nginx2") By("Waiting for all pods to be running") - expectNumberOfPodsRunning(6) + expectNumberOfPodsRunning(2) By("Cordening the worker2 now it has workload on it") utils.CordonNode(worker2Node) @@ -404,24 +367,7 @@ spec: By("Uncordening the other node") utils.UncordonNode(worker3Node) - By("Creating a nodeDrain for our node") - nodeDrain := fmt.Sprintf(` -apiVersion: k8s.gezb.co.uk/v1 -kind: NodeDrain -metadata: - name: nodedrain-sample -spec: - nodeName: %s - ignoreVersion: true - disableCordon: true # manually cordening nodes - -`, worker2Node) - - nodeDrainFile, err := utils.CreateTempFile(nodeDrain) - Expect(err).NotTo(HaveOccurred(), "Failed apply nodeDrain") - cmd := exec.Command("kubectl", "apply", "-f", nodeDrainFile) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed apply nodeDrain") + nodeDrainFile := applyNodeDrain(false) nodeDrainIsPhaseOtherNodesNotCordoned := func(g Gomega) { cmd := exec.Command("kubectl", "get", "nodedrain", "nodedrain-sample") @@ -432,12 +378,12 @@ spec: Eventually(nodeDrainIsPhaseOtherNodesNotCordoned, 5*time.Minute).Should(Succeed()) By("Cordening worker3") - cmd = exec.Command("kubectl", "cordon", worker3Node) - _, err = utils.Run(cmd) + cmd := exec.Command("kubectl", "cordon", worker3Node) + _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed delete nodeDrain") - By("Drain should run and we should be left with only deamonsets") - expectNumberOfPodsRunning(4) + By("Drain should run and we should be left no pods") + expectNumberOfPodsRunning(0) nodeDrainIsPhaseCompleted := func(g Gomega) { cmd := exec.Command("kubectl", "get", "nodedrain", "nodedrain-sample") @@ -447,6 +393,8 @@ spec: } Eventually(nodeDrainIsPhaseCompleted, 5*time.Minute).Should(Succeed()) + By("Clean up after the test") + By(" Deleting nginx statefulsets") cmd = exec.Command("kubectl", "delete", "statefulset", "nginx") _, err = utils.Run(cmd) @@ -467,6 +415,7 @@ spec: By("re-cordening our test nodes") utils.CordonNode(worker2Node) utils.CordonNode(worker3Node) + }) It("if configured will wait for evicted pods to get to running state", func() { @@ -476,35 +425,18 @@ spec: By("Verifying the number of pods on the node(s)") - expectNumberOfPodsRunning(4) + expectNumberOfPodsRunning(0) createStatefulSetWithName("nginx") createStatefulSetWithName("nginx2") By("Waiting for all pods to be running") - expectNumberOfPodsRunning(6) + expectNumberOfPodsRunning(2) - By("Creating a nodeDrain for our node") - nodeDrain := fmt.Sprintf(` -apiVersion: k8s.gezb.co.uk/v1 -kind: NodeDrain -metadata: - name: nodedrain-sample -spec: - nodeName: %s - ignoreVersion: true - waitForPodsToRestart: true - -`, worker2Node) - - nodeDrainFile, err := utils.CreateTempFile(nodeDrain) - Expect(err).NotTo(HaveOccurred(), "Failed apply nodeDrain") - cmd := exec.Command("kubectl", "apply", "-f", nodeDrainFile) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed apply nodeDrain") + nodeDrainFile := applyNodeDrain(true) - By("Drain should run and we should be left with only deamonsets") - expectNumberOfPodsRunning(4) + By("Drain should run and we should be left no pods") + expectNumberOfPodsRunning(0) nodeDrainIsPhaseWaitForPodsToRestart := func(g Gomega) { cmd := exec.Command("kubectl", "get", "nodedrain", "nodedrain-sample") @@ -526,9 +458,11 @@ spec: } Eventually(nodeDrainIsPhaseCompleted, 5*time.Minute).Should(Succeed()) + By("Clean up after the test") + By(" Deleting nginx statefulsets") - cmd = exec.Command("kubectl", "delete", "statefulset", "nginx") - _, err = utils.Run(cmd) + cmd := exec.Command("kubectl", "delete", "statefulset", "nginx") + _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed delete nginx statefulset") By(" Deleting nginx2 statefulsets") @@ -548,6 +482,51 @@ spec: }) }) +func applyDrainCheck(podRegex string) string { + By("Creating DrainCheck for:" + podRegex) + + drainCheck := fmt.Sprintf(` +apiVersion: k8s.gezb.co.uk/v1 +kind: DrainCheck +metadata: + name: draincheck-sample +spec: + podRegex: %s +`, podRegex) + + drainCheckFile, err := utils.CreateTempFile(drainCheck) + Expect(err).NotTo(HaveOccurred(), "Failed creating DrainCheck file to apply: "+drainCheckFile) + + cmd := exec.Command("kubectl", "apply", "-f", drainCheckFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed apply DrainCheck") + return drainCheckFile +} + +func applyNodeDrain(waitForPodToRestart bool) string { + By("Creating NodeDrain for node :" + worker2Node) + + nodeDrain := fmt.Sprintf(` +apiVersion: k8s.gezb.co.uk/v1 +kind: NodeDrain +metadata: + name: nodedrain-sample +spec: + nodeName: %s + versionToDrainRegex: %s + nodeRole: %s + waitForPodsToRestart: %t +`, worker2Node, K8sVersionRegex, k8sRole, waitForPodToRestart) + + nodeDrainFile, err := utils.CreateTempFile(nodeDrain) + Expect(err).NotTo(HaveOccurred(), "Failed creating NodeDrain file to apply: "+nodeDrainFile) + + cmd := exec.Command("kubectl", "apply", "-f", nodeDrainFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed apply NodeDrain") + return nodeDrainFile +} + func expectNumberOfPodsRunning(expected int) { var worker2, worker3 []string verifyAllPodsRunning := func(g Gomega) { @@ -577,7 +556,7 @@ func expectNumberOfPodsRunning(expected int) { } g.Expect(len(worker2) + len(worker3)).To(Equal(expected)) } - EventuallyWithOffset(-2, verifyAllPodsRunning, 3*time.Minute).Should(Succeed()) + EventuallyWithOffset(-2, verifyAllPodsRunning, time.Minute).Should(Succeed()) } func createStatefulSetWithName(name string) { diff --git a/test/utils/utils.go b/test/utils/utils.go index d6a8b28..a66f0bf 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -162,6 +162,14 @@ func LoadImageToKindClusterWithName(name string) error { return err } +// LoadImageToK3dClusterWithName loads a local docker image to the k3d cluster +func LoadImageToK3dClusterWithName(clusterName, imageName string) error { + k3dOptions := []string{"image", "import", imageName, "--cluster", clusterName} + cmd := exec.Command("k3d", k3dOptions...) + _, err := Run(cmd) + return err +} + // GetNonEmptyLines converts given command output string into individual objects // according to line breakers, and ignores the empty elements in it. func GetNonEmptyLines(output string) []string { @@ -258,6 +266,13 @@ func LabelNode(nodeName string, role string) { ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to label node "+role) } +func DrainNodes(nodeNames []string) { + args := append([]string{"drain", "--ignore-daemonsets", "--delete-emptydir-data"}, nodeNames...) + cmd := exec.Command("kubectl", args...) + _, err := Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to drain nodes "+strings.Join(nodeNames, ", ")) +} + func CordonNode(nodeName string) { cmd := exec.Command("kubectl", "cordon", nodeName) _, err := Run(cmd)