perfprof: enable exec-cpu-affinity by default

For High-performance configuration, cri-o started supporting exec-cpu-affinity
feature and when configured to `first` it provides the ability for exec
process to be pinned to the first CPU from the shared-CPUs IF set
or to the first one from the islolated set.
(see cri-o/cri-o@4dd7fb9)

In performance profile, we want to enable this high-performance feature
by default, and disbale it (legacy) it provides an annotation option.

The annotation is there just as a backup in case of bugs getting
reported by the consequences of this feature enablement, and should be
removed in 2 releases.

Run `./hack/render-sync.sh` to update the (no-cluster) e2e tests
expected outputs.

Signed-off-by: Shereen Haj <shajmakh@redhat.com>

Author: shajmakh

assets/performanceprofile/configs/99-runtimes.conf

@@ -11,4 +11,5 @@ infra_ctr_cpuset = "{{.ReservedCpus}}"
 # do not have high-performance binary under the $PATH that will point to it.
 [crio.runtime.runtimes.high-performance]
 inherit_default_runtime = true
+{{if .ExecCPUAffinity}}exec_cpu_affinity = "{{.ExecCPUAffinity}}"{{end}}
 allowed_annotations = ["cpu-load-balancing.crio.io", "cpu-quota.crio.io", "irq-load-balancing.crio.io", "cpu-c-states.crio.io", "cpu-freq-governor.crio.io"{{ if .CrioSharedCPUsAnnotation }}{{ printf ", %q" .CrioSharedCPUsAnnotation}}{{end}}]

pkg/apis/performanceprofile/v2/performanceprofile_types.go

@@ -34,6 +34,11 @@ const PerformanceProfileEnablePhysicalRpsAnnotation = "performance.openshift.io/
 // Valid values: "true", "enable" (to enable), "false", "disable" (to disable).
 const PerformanceProfileEnableRpsAnnotation = "performance.openshift.io/enable-rps"
 
+// PerformanceProfileDisableExecCPUAffinityAnnotation disables the exec-cpu-affinity setting for the node.
+// Valid values: "true". Other values are ignored and treated as "false".
+// remove in 4.23
+const PerformanceProfileDisableExecCPUAffinityAnnotation = "performance.openshift.io/disable-exec-cpu-affinity"
+
 // PerformanceProfileSpec defines the desired state of PerformanceProfile.
 type PerformanceProfileSpec struct {
 	// CPU defines a set of CPU related parameters.

pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go

@@ -118,6 +118,7 @@ const (
 	templateOvsSliceUsageFile        = "01-use-ovs-slice.conf"
 	templateWorkload                 = "Workload"
 	templateCrioSharedCPUsAnnotation = "CrioSharedCPUsAnnotation"
+	templateExecCPUAffinity          = "ExecCPUAffinity"
 )
 
 // New returns new machine configuration object for performance sensitive workloads
@@ -587,6 +588,10 @@ func renderCrioConfigSnippet(profile *performancev2.PerformanceProfile, src stri
 		templateArgs[templateReservedCpus] = string(*profile.Spec.CPU.Reserved)
 	}
 
+	if profilecomponent.IsExecCPUAffinityEnabled(profile) {
+		templateArgs[templateExecCPUAffinity] = "first"
+	}
+
 	if opts.MixedCPUsEnabled {
 		templateArgs[templateSharedCpus] = string(*profile.Spec.CPU.Shared)
 		templateArgs[templateCrioSharedCPUsAnnotation] = "cpu-shared.crio.io"

pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -141,6 +142,58 @@ var _ = Describe("Machine Config", func() {
 		})
 	})
 
+	Context("machine config creation with CRI-O runtime config", func() {
+		crioRuntimeConfigPath := filepath.Join(crioConfd, crioRuntimesConfig)
+
+		It("should create machine config with exec-cpu-affinity set by default", func() {
+			profile := testutils.NewPerformanceProfile("test")
+
+			mc, err := New(profile, &components.MachineConfigOptions{})
+			Expect(err).ToNot(HaveOccurred())
+
+			result := igntypes.Config{}
+			Expect(json.Unmarshal(mc.Spec.Config.Raw, &result)).To(Succeed())
+
+			var content string
+			for _, f := range result.Storage.Files {
+				if f.Path == crioRuntimeConfigPath {
+					base64Data := strings.TrimPrefix(*f.Contents.Source, "data:text/plain;charset=utf-8;base64,")
+					decoded, err := base64.StdEncoding.DecodeString(base64Data)
+					Expect(err).ToNot(HaveOccurred())
+					content = string(decoded)
+					break
+				}
+			}
+			Expect(content).ToNot(BeEmpty(), "crio runtime config not found")
+			Expect(content).To(ContainSubstring("exec_cpu_affinity = \"first\""))
+		})
+
+		It("should create machine config without exec-cpu-affinity when annotation is set to true", func() {
+			profile := testutils.NewPerformanceProfile("test")
+			profile.Annotations = map[string]string{}
+			profile.Annotations[performancev2.PerformanceProfileDisableExecCPUAffinityAnnotation] = "true"
+
+			mc, err := New(profile, &components.MachineConfigOptions{})
+			Expect(err).ToNot(HaveOccurred())
+
+			result := igntypes.Config{}
+			Expect(json.Unmarshal(mc.Spec.Config.Raw, &result)).To(Succeed())
+
+			var content string
+			for _, f := range result.Storage.Files {
+				if f.Path == crioRuntimeConfigPath {
+					base64Data := strings.TrimPrefix(*f.Contents.Source, "data:text/plain;charset=utf-8;base64,")
+					decoded, err := base64.StdEncoding.DecodeString(base64Data)
+					Expect(err).ToNot(HaveOccurred())
+					content = string(decoded)
+					break
+				}
+			}
+			Expect(content).ToNot(BeEmpty(), "crio runtime config not found")
+			Expect(content).To(Not(ContainSubstring("exec_cpu_affinity = \"first\"")))
+		})
+	})
+
 	Context("machine config creation with enabled RPS using alternative values", func() {
 		It("should create machine config with RPS configuration when enabled with 'enable'", func() {
 			profile := testutils.NewPerformanceProfile("test")

pkg/performanceprofile/controller/performanceprofile/components/profile/profile.go

@@ -94,3 +94,17 @@ func IsMixedCPUsEnabled(profile *performancev2.PerformanceProfile) bool {
 	}
 	return *profile.Spec.WorkloadHints.MixedCpus
 }
+
+// IsExecCPUAffinityEnabled checks if exec-cpu-affinity feature should be enabled
+func IsExecCPUAffinityEnabled(profile *performancev2.PerformanceProfile) bool {
+	if profile.Annotations != nil {
+		isExecCPUAffinityDisabled, ok := profile.Annotations[performancev2.PerformanceProfileDisableExecCPUAffinityAnnotation]
+		if ok && isExecCPUAffinityDisabled == "true" {
+			// run the legacy behavior and disable exec-cpu-affinity
+			return false
+		}
+	}
+
+	// The default behavior is to enable exec-cpu-affinity whenever profile is applied
+	return true
+}

test/e2e/performanceprofile/functests-performance-profile-creator/1_performance-profile_creator/1_performance-profile_creator.test

@@ -43,7 +43,7 @@ spec:
         path: /usr/local/bin/clear-irqbalance-banned-cpus.sh
         user: {}
       - contents:
-          source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K
+          source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQpleGVjX2NwdV9hZmZpbml0eSA9ICJmaXJzdCIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg==
           verification: {}
         group: {}
         mode: 420

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -43,7 +43,7 @@ spec:
         path: /usr/local/bin/clear-irqbalance-banned-cpus.sh
         user: {}
       - contents:
-          source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K
+          source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQpleGVjX2NwdV9hZmZpbml0eSA9ICJmaXJzdCIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg==
           verification: {}
         group: {}
         mode: 420

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml

@@ -43,7 +43,7 @@ spec:
         path: /usr/local/bin/clear-irqbalance-banned-cpus.sh
         user: {}
       - contents:
-          source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K
+          source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi03IgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQpleGVjX2NwdV9hZmZpbml0eSA9ICJmaXJzdCIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg==
           verification: {}
         group: {}
         mode: 420

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -43,7 +43,7 @@ spec:
         path: /usr/local/bin/clear-irqbalance-banned-cpus.sh
         user: {}
       - contents:
-          source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQphbGxvd2VkX2Fubm90YXRpb25zID0gWyJjcHUtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtcXVvdGEuY3Jpby5pbyIsICJpcnEtbG9hZC1iYWxhbmNpbmcuY3Jpby5pbyIsICJjcHUtYy1zdGF0ZXMuY3Jpby5pbyIsICJjcHUtZnJlcS1nb3Zlcm5vci5jcmlvLmlvIl0K
+          source: data:text/plain;charset=utf-8;base64,CltjcmlvLnJ1bnRpbWVdCmluZnJhX2N0cl9jcHVzZXQgPSAiMi0zIgoKCgoKIyBUaGUgQ1JJLU8gd2lsbCBjaGVjayB0aGUgYWxsb3dlZF9hbm5vdGF0aW9ucyB1bmRlciB0aGUgcnVudGltZSBoYW5kbGVyIGFuZCBhcHBseSBoaWdoLXBlcmZvcm1hbmNlIGhvb2tzIHdoZW4gb25lIG9mCiMgaGlnaC1wZXJmb3JtYW5jZSBhbm5vdGF0aW9ucyBwcmVzZW50cyB1bmRlciBpdC4KIyBXZSBzaG91bGQgcHJvdmlkZSB0aGUgcnVudGltZV9wYXRoIGJlY2F1c2Ugd2UgbmVlZCB0byBpbmZvcm0gdGhhdCB3ZSB3YW50IHRvIHJlLXVzZSBydW5jIGJpbmFyeSBhbmQgd2UKIyBkbyBub3QgaGF2ZSBoaWdoLXBlcmZvcm1hbmNlIGJpbmFyeSB1bmRlciB0aGUgJFBBVEggdGhhdCB3aWxsIHBvaW50IHRvIGl0LgpbY3Jpby5ydW50aW1lLnJ1bnRpbWVzLmhpZ2gtcGVyZm9ybWFuY2VdCmluaGVyaXRfZGVmYXVsdF9ydW50aW1lID0gdHJ1ZQpleGVjX2NwdV9hZmZpbml0eSA9ICJmaXJzdCIKYWxsb3dlZF9hbm5vdGF0aW9ucyA9IFsiY3B1LWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LXF1b3RhLmNyaW8uaW8iLCAiaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW8iLCAiY3B1LWMtc3RhdGVzLmNyaW8uaW8iLCAiY3B1LWZyZXEtZ292ZXJub3IuY3Jpby5pbyJdCg==
           verification: {}
         group: {}
         mode: 420