e2e: PP: cover ExecCPUAffinity support in tests

Add basic e2e tests that checks the default behavior of
performance-profile with default enabled `ExecCPUAffinity: first`.

Signed-off-by: Shereen Haj <shajmakh@redhat.com>

Author: shajmakh

test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"strconv"
 	"strings"
 	"time"
@@ -510,6 +511,131 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() {
 			})
 		})
 	})
+
+	Context("Check exec-cpu-affinity feature", func() {
+		When("exec-cpu-affinity is enabled (default in PP)", func() {
+			var workerRTNode *corev1.Node
+			var profile *performancev2.PerformanceProfile
+			var getter cgroup.ControllersGetter
+			var updatedShared, updatedIsolated cpuset.CPUSet
+
+			BeforeEach(func() {
+				By("Checking if exec-cpu-affinity is enabled by default in the profile")
+				profile, _ = profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
+				Expect(profile).ToNot(BeNil(), "Failed to get performance profile")
+				if profile.Annotations != nil {
+					val, ok := profile.Annotations[performancev2.PerformanceProfileDisableExecCPUAffinityAnnotation]
+					if ok && val == "true" {
+						// fail loadly because the default should be enabled
+						Fail("exec-cpu-affinity is disabled in the profile")
+					}
+				}
+
+				By("Updating performance profile to have enough shared cpus if needed")
+				updatedIsolated = *mustParse(string(*profile.Spec.CPU.Isolated))
+				currentShared := mustParse(string(*profile.Spec.CPU.Shared))
+				if len(currentShared.List()) < 2 {
+					testlog.Info("shared cpuset has less than 2 cpus; this test requires at least 2 shared cpus; update the profile")
+					isolated := mustParse(string(*profile.Spec.CPU.Isolated))
+
+					// we need 4 total isolated and shared CPUs:
+					// 1 as a buffer for node's base load
+					// 1 as the test gu pod requests
+					// 2 as shared cpus
+					leastIsolatedCpus := 3
+					if len(currentShared.List()) == 0 {
+						leastIsolatedCpus = 4
+					}
+					if len(isolated.List()) < leastIsolatedCpus {
+						Skip(fmt.Sprintf("isolated cpuset has less than %d cpus; this test requires at least %d isolated cpus", leastIsolatedCpus, leastIsolatedCpus))
+					}
+
+					updatedShared = cpuset.New(isolated.List()[0], isolated.List()[1])
+					updatedIsolated = cpuset.New(isolated.List()[2:]...)
+
+					if len(currentShared.List()) == 1 {
+						updatedShared = cpuset.New(currentShared.List()[0], isolated.List()[0])
+						updatedIsolated = cpuset.New(isolated.List()[1:]...)
+					}
+
+					testlog.Infof("shared cpu ids to be updated are: %q", updatedShared.String())
+					profile.Spec.CPU.Isolated = cpuSetToPerformanceCPUSet(&updatedIsolated)
+					profile.Spec.CPU.Shared = cpuSetToPerformanceCPUSet(&updatedShared)
+					profile.Spec.WorkloadHints.MixedCpus = ptr.To(true) // if not already
+
+					profiles.UpdateWithRetry(profile)
+
+					poolName := poolname.GetByProfile(context.TODO(), profile)
+					By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName))
+					profilesupdate.WaitForTuningUpdating(context.TODO(), profile)
+					By(fmt.Sprintf("Waiting when %s finishes updates", poolName))
+					profilesupdate.WaitForTuningUpdated(context.TODO(), profile)
+				}
+
+				workerRTNodes, err := nodes.GetByLabels(testutils.NodeSelectorLabels)
+				Expect(err).ToNot(HaveOccurred())
+				workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes)
+				Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("error looking for the optional selector: %v", err))
+				Expect(workerRTNodes).ToNot(BeEmpty())
+				workerRTNode = &workerRTNodes[0]
+
+				getter, err = cgroup.BuildGetter(ctx, testclient.DataPlaneClient, testclient.K8sClient)
+				Expect(err).ToNot(HaveOccurred())
+			})
+
+			It("should pin exec process to first Shared CPU of the container - guratanteed pod", func() {
+				By("Creating a guaranteed test pod with shared CPU request")
+				rl := &corev1.ResourceList{
+					corev1.ResourceCPU:    resource.MustParse("1"),
+					corev1.ResourceMemory: resource.MustParse("100Mi"),
+					sharedCpusResource:    resource.MustParse("1"),
+				}
+				testPod := makePod(ctx, testclient.DataPlaneClient, testutils.NamespaceTesting,
+					withRequests(rl),
+					withLimits(rl),
+					onNode(workerRTNode.Name),
+					withRuntime(components.GetComponentName(profile.Name, components.ComponentNamePrefix)))
+
+				Expect(testclient.Client.Create(ctx, testPod)).To(Succeed(), "Failed to create test pod")
+				testPod, err := pods.WaitForCondition(ctx, client.ObjectKeyFromObject(testPod), corev1.PodReady, corev1.ConditionTrue, 5*time.Minute)
+				Expect(err).ToNot(HaveOccurred())
+				defer func() {
+					if testPod != nil {
+						testlog.Infof("deleting pod %q", testPod.Name)
+						Expect(pods.Delete(ctx, testPod)).To(BeTrue(), "Failed to delete pod")
+					}
+				}()
+
+				By("Prepare comparable data")
+				cpusetCfg := &controller.CpuSet{}
+				Expect(getter.Container(ctx, testPod, testPod.Spec.Containers[0].Name, cpusetCfg)).To(Succeed(), "Failed to get cpuset config for test pod")
+
+				cpusIncludingShared, err := cpuset.Parse(cpusetCfg.Cpus)
+				testlog.Infof("cpus including shared: %s", cpusIncludingShared.String())
+				Expect(err).ToNot(HaveOccurred(), "Failed to parse cpuset config for test pod cpus=%q", cpusetCfg.Cpus)
+				cntShared := cpusIncludingShared.Difference(updatedIsolated)
+				firstSharedCPU := cntShared.List()[0]
+				testlog.Infof("first shared CPU: %d; all shared CPUs: %s", firstSharedCPU, cntShared.String())
+
+				retries := int(math.Ceil(float64(10) / float64(cntShared.Size())))
+				By("Run exec command on the pod and verify the process is pinned to the first shared CPU")
+
+				for i := 0; i < retries; i++ {
+					cmd := []string{"/bin/bash", "-c", "sleep 10 & SLPID=$!; ps -o psr -p $SLPID;"}
+					output, err := pods.ExecCommandOnPod(testclient.K8sClient, testPod, testPod.Spec.Containers[0].Name, cmd)
+					Expect(err).ToNot(HaveOccurred(), "Failed to exec command on the pod; retry %d", i)
+					strout := string(output)
+					testlog.Infof("retry %d exec command output: %s", i, strout)
+
+					strout = strings.ReplaceAll(strout, "PSR", "")
+					execProcessCPUs := strings.TrimSpace(strout)
+					testlog.Infof("exec process CPU: %s", execProcessCPUs)
+					Expect(execProcessCPUs).ToNot(BeEmpty(), "Failed to get exec process CPU; retry %d", i)
+					Expect(execProcessCPUs).To(Equal(firstSharedCPU), "Exec process CPU is not the first shared CPU; retry %d", i)
+				}
+			})
+		})
+	})
 })
 
 func setup(ctx context.Context) func(ctx2 context.Context) {
@@ -659,6 +785,12 @@ func withRuntime(name string) func(p *corev1.Pod) {
 	}
 }
 
+func onNode(nodeName string) func(p *corev1.Pod) {
+	return func(p *corev1.Pod) {
+		p.Spec.NodeName = nodeName
+	}
+}
+
 func getTestingNamespace() corev1.Namespace {
 	return *namespaces.TestingNamespace
 }

test/e2e/performanceprofile/functests/1_performance/cpu_management.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"os"
 	"regexp"
 	"strconv"
@@ -12,7 +13,6 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -248,7 +248,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 		})
 
 		AfterEach(func() {
-			deleteTestPod(context.TODO(), testpod)
+			Expect(pods.Delete(context.TODO(), testpod)).To(BeTrue(), "Failed to delete pod")
 		})
 
 		DescribeTable("Verify CPU usage by stress PODs", func(ctx context.Context, guaranteed bool) {
@@ -339,7 +339,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 		AfterEach(func() {
-			deleteTestPod(context.TODO(), testpod)
+			Expect(pods.Delete(context.TODO(), testpod)).To(BeTrue(), "Failed to delete pod")
 		})
 		When("kubelet is restart", func() {
 			It("[test_id: 73501] defaultCpuset should not change", func() {
@@ -422,7 +422,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 
 		AfterEach(func() {
 			if testpod != nil {
-				deleteTestPod(context.TODO(), testpod)
+				Expect(pods.Delete(context.TODO(), testpod)).To(BeTrue(), "Failed to delete pod")
 			}
 		})
 
@@ -481,7 +481,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 				fmt.Sprintf("IRQ still active on CPU%s", psr))
 
 			By("Checking that after removing POD default smp affinity is returned back to all active CPUs")
-			deleteTestPod(context.TODO(), testpod)
+			Expect(pods.Delete(context.TODO(), testpod)).To(BeTrue(), "Failed to delete pod")
 			defaultSmpAffinitySet, err = nodes.GetDefaultSmpAffinitySet(context.TODO(), workerRTNode)
 			Expect(err).ToNot(HaveOccurred())
 
@@ -580,7 +580,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 			if testpod == nil {
 				return
 			}
-			deleteTestPod(context.TODO(), testpod)
+			Expect(pods.Delete(context.TODO(), testpod)).To(BeTrue(), "Failed to delete pod")
 		})
 
 		It("[test_id:49149] should reject pods which request integral CPUs not aligned with machine SMT level", func() {
@@ -633,7 +633,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 			if testpod == nil {
 				return
 			}
-			deleteTestPod(context.TODO(), testpod)
+			Expect(pods.Delete(context.TODO(), testpod)).To(BeTrue(), "Failed to delete pod")
 		})
 
 		DescribeTable("Verify Hyper-Thread aware scheduling for guaranteed pods",
@@ -680,7 +680,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 					testpod = startHTtestPod(ctx, cpuCount)
 					Expect(checkPodHTSiblings(ctx, testpod)).To(BeTrue(), "Pod cpu set does not map to host cpu sibling pairs")
 					By("Deleting test pod...")
-					deleteTestPod(ctx, testpod)
+					Expect(pods.Delete(ctx, testpod)).To(BeTrue(), "Failed to delete pod")
 				}
 			},
 
@@ -983,7 +983,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 			defer func() {
 				if guaranteedPod != nil {
 					testlog.Infof("deleting pod %q", guaranteedPod.Name)
-					deleteTestPod(ctx, guaranteedPod)
+					Expect(pods.Delete(ctx, guaranteedPod)).To(BeTrue(), "Failed to delete guaranteed pod")
 				}
 			}()
 
@@ -1014,7 +1014,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 			defer func() {
 				if bestEffortPod != nil {
 					testlog.Infof("deleting pod %q", bestEffortPod.Name)
-					deleteTestPod(ctx, bestEffortPod)
+					Expect(pods.Delete(ctx, bestEffortPod)).To(BeTrue(), "Failed to delete best-effort pod")
 				}
 			}()
 
@@ -1142,15 +1142,73 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
 			defer func() {
 				if guPod != nil {
 					testlog.Infof("deleting pod %q", guPod.Name)
-					deleteTestPod(ctx, guPod)
+					Expect(pods.Delete(ctx, guPod)).To(BeTrue(), "Failed to delete guaranteed pod")
 				}
 				if buPod != nil {
 					testlog.Infof("deleting pod %q", buPod.Name)
-					deleteTestPod(ctx, buPod)
+					Expect(pods.Delete(ctx, buPod)).To(BeTrue(), "Failed to delete burstable pod")
 				}
 			}()
 		})
 	})
+
+	Context("Check exec-cpu-affinity feature", func() {
+		When("exec-cpu-affinity is enabled (default in PP)", func() {
+			// shared-cpus case is covered in 11_mixedcpus test
+			BeforeEach(func() {
+				By("Checking if exec-cpu-affinity is enabled by default in the profile")
+				profile, _ := profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
+				Expect(profile).ToNot(BeNil(), "Failed to get performance profile")
+				if profile.Annotations != nil {
+					val, ok := profile.Annotations[performancev2.PerformanceProfileDisableExecCPUAffinityAnnotation]
+					if ok && val == "true" {
+						// fail loadly because the default should be enabled
+						Fail("exec-cpu-affinity is disabled in the profile")
+					}
+				}
+			})
+
+			It("should pin exec process to first CPU dedicated to the container - guratanteed pod", func() {
+				By("Creating a guaranteed test pod")
+				testPod := makePod(ctx, workerRTNode, true)
+				Expect(testclient.Client.Create(ctx, testPod)).To(Succeed(), "Failed to create test pod")
+				testPod, err = pods.WaitForCondition(ctx, client.ObjectKeyFromObject(testPod), corev1.PodReady, corev1.ConditionTrue, 5*time.Minute)
+				Expect(err).ToNot(HaveOccurred())
+				defer func() {
+					if testPod != nil {
+						testlog.Infof("deleting pod %q", testPod.Name)
+						Expect(pods.Delete(ctx, testPod)).To(BeTrue(), "Failed to delete test pod")
+					}
+				}()
+
+				cpusetCfg := &controller.CpuSet{}
+				Expect(getter.Container(ctx, testPod, testPod.Spec.Containers[0].Name, cpusetCfg)).To(Succeed(), "Failed to get cpuset config for test pod")
+
+				cpusList := strings.Split(cpusetCfg.Cpus, ",")
+				Expect(cpusList).ToNot(BeEmpty())
+				// assumes no shared configured in this suite
+				firstExclusiveCPU := strings.TrimSpace(cpusList[0])
+				testlog.Infof("first exclusive CPU: %s, all exclusive CPUs: %s", firstExclusiveCPU, strings.Join(cpusList, ","))
+
+				cpuRequest := testPod.Spec.Containers[0].Resources.Requests.Name(corev1.ResourceCPU, resource.DecimalSI).Value()
+				retries := int(math.Ceil(float64(10) / float64(cpuRequest)))
+				By("Run exec command on the pod and verify the process is pinned to the first exclusive CPU")
+
+				for i := 0; i < retries; i++ {
+					cmd := []string{"/bin/bash", "-c", "sleep 10 & SLPID=$!; ps -o psr -p $SLPID;"}
+					output, err := pods.ExecCommandOnPod(testclient.K8sClient, testPod, testPod.Spec.Containers[0].Name, cmd)
+					Expect(err).ToNot(HaveOccurred(), "Failed to exec command on the pod; retry %d", i)
+					strout := string(output)
+					testlog.Infof("retry %d exec command output: %s", i, strout)
+
+					strout = strings.ReplaceAll(strout, "PSR", "")
+					execProcessCPUs := strings.TrimSpace(strout)
+					Expect(execProcessCPUs).ToNot(BeEmpty(), "Failed to get exec process CPU; retry %d", i)
+					Expect(execProcessCPUs).To(Equal(firstExclusiveCPU), "Exec process CPU is not the first exclusive CPU; retry %d", i)
+				}
+			})
+		})
+	})
 })
 
 func extractConfigInfo(output string) (*ContainerConfig, error) {
@@ -1401,24 +1459,6 @@ func getTestPodWithAnnotations(annotations map[string]string, cpus int) *corev1.
 	return testpod
 }
 
-func deleteTestPod(ctx context.Context, testpod *corev1.Pod) (types.UID, bool) {
-	// it possible that the pod already was deleted as part of the test, in this case we want to skip teardown
-	err := testclient.DataPlaneClient.Get(ctx, client.ObjectKeyFromObject(testpod), testpod)
-	if errors.IsNotFound(err) {
-		return "", false
-	}
-
-	testpodUID := testpod.UID
-
-	err = testclient.DataPlaneClient.Delete(ctx, testpod)
-	Expect(err).ToNot(HaveOccurred())
-
-	err = pods.WaitForDeletion(ctx, testpod, pods.DefaultDeletionTimeout*time.Second)
-	Expect(err).ToNot(HaveOccurred())
-
-	return testpodUID, true
-}
-
 func cpuSpecToString(cpus *performancev2.CPU) (string, error) {
 	if cpus == nil {
 		return "", fmt.Errorf("performance CPU field is nil")

test/e2e/performanceprofile/functests/1_performance/irqbalance.go

@@ -213,7 +213,7 @@ var _ = Describe("[performance] Checking IRQBalance settings", Ordered, func() {
 			defer func() {
 				if testpod != nil {
 					testlog.Infof("deleting pod %q", testpod.Name)
-					deleteTestPod(context.TODO(), testpod)
+					Expect(pods.Delete(context.TODO(), testpod)).To(BeTrue(), "Failed to delete pod")
 				}
 				bannedCPUs, err := getIrqBalanceBannedCPUs(context.TODO(), targetNode)
 				Expect(err).ToNot(HaveOccurred(), "failed to extract the banned CPUs from node %q", targetNode.Name)

test/e2e/performanceprofile/functests/utils/pods/pods.go

@@ -20,6 +20,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client"
@@ -51,6 +52,28 @@ func GetTestPod() *corev1.Pod {
 	}
 }
 
+func Delete(ctx context.Context, pod *corev1.Pod) bool {
+	err := testclient.DataPlaneClient.Get(ctx, client.ObjectKeyFromObject(pod), pod)
+	if errors.IsNotFound(err) {
+		klog.InfoS("pod already deleted", "namespace", pod.Namespace, "name", pod.Name)
+		return true
+	}
+
+	err = testclient.DataPlaneClient.Delete(ctx, pod)
+	if err != nil {
+		klog.ErrorS(err, "failed to delete pod", "namespace", pod.Namespace, "name", pod.Name)
+		return false
+	}
+
+	err = WaitForDeletion(ctx, pod, DefaultDeletionTimeout*time.Second)
+	if err != nil {
+		klog.ErrorS(err, "failed to wait for pod deletion", "namespace", pod.Namespace, "name", pod.Name)
+		return false
+	}
+
+	return true
+}
+
 // WaitForDeletion waits until the pod will be removed from the cluster
 func WaitForDeletion(ctx context.Context, pod *corev1.Pod, timeout time.Duration) error {
 	return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) {